Before 3d models

src/main/java/com/elphel/imagej/cameras/CLTParameters.java

@@ -376,6 +376,11 @@ public class CLTParameters {
 	public double     photo_min_strength =            0.0;   // maybe add to filter out weak tiles
 	public double     photo_max_diff =                40.0;  // To filter mismatches. Normal (adjusted) have RMSE ~9
 	public int        photo_order =                   2;     // Approximation order: 0 - just offset, 1 - linear, 2 - quadratic
+	public double     photo_std_1 =                  75.0;   // Minimal standard deviation of the filtered values for poly order 1
+	public double     photo_std_2 =                 200.0;   // Minimal standard deviation of the filtered values for poly order 2
+	public int        photo_offs_set =                0;     // 0 - keep weighted offset average, 1 - balance result image, 2 - set weighted average to specific value
+	public double     photo_offs =                 21946;    // weighted average offset target value, if photo_offs_set (and not photo_offs_balance)
+	
 	public boolean    photo_debug =                   false; // Generate images and text
 	
 	
@@ -1367,6 +1372,10 @@ public class CLTParameters {
 		properties.setProperty(prefix+"photo_min_strength",         this.photo_min_strength+"");  // double
 		properties.setProperty(prefix+"photo_max_diff",             this.photo_max_diff+"");      // double
 		properties.setProperty(prefix+"photo_order",                this.photo_order+"");         // int
+		properties.setProperty(prefix+"photo_std_1",                this.photo_std_1+"");         // double
+		properties.setProperty(prefix+"photo_std_2",                this.photo_std_2+"");         // double
+		properties.setProperty(prefix+"photo_offs_set",             this.photo_offs_set+"");      // int
+		properties.setProperty(prefix+"photo_offs",                 this.photo_offs+"");          // double
 		properties.setProperty(prefix+"photo_debug",                this.photo_debug+"");         // boolean
 		
 		properties.setProperty(prefix+"show_textures",              this.show_textures+"");
@@ -2234,6 +2243,10 @@ public class CLTParameters {
 		if (properties.getProperty(prefix+"photo_min_strength")!=null)   this.photo_min_strength=Double.parseDouble(properties.getProperty(prefix+"photo_min_strength"));
 		if (properties.getProperty(prefix+"photo_max_diff")!=null)       this.photo_max_diff=Double.parseDouble(properties.getProperty(prefix+"photo_max_diff"));
 		if (properties.getProperty(prefix+"photo_order")!=null)          this.photo_order=Integer.parseInt(properties.getProperty(prefix+"photo_order"));		
+		if (properties.getProperty(prefix+"photo_std_1")!=null)          this.photo_std_1=Double.parseDouble(properties.getProperty(prefix+"photo_std_1"));
+		if (properties.getProperty(prefix+"photo_std_2")!=null)          this.photo_std_2=Double.parseDouble(properties.getProperty(prefix+"photo_std_2"));
+		if (properties.getProperty(prefix+"photo_offs_set")!=null)       this.photo_offs_set=Integer.parseInt(properties.getProperty(prefix+"photo_offs_set"));		
+		if (properties.getProperty(prefix+"photo_offs")!=null)           this.photo_offs=Double.parseDouble(properties.getProperty(prefix+"photo_offs"));
 		if (properties.getProperty(prefix+"photo_debug")!=null)          this.photo_debug=Boolean.parseBoolean(properties.getProperty(prefix+"photo_debug"));		
 		
 		if (properties.getProperty(prefix+"show_textures")!=null)                 this.show_textures=Boolean.parseBoolean(properties.getProperty(prefix+"show_textures"));
@@ -3245,12 +3258,18 @@ public class CLTParameters {
 
 		gd.addNumericField("Approximation polynomial order",         this.photo_order, 0,3,"",
 				"0 - only offset, 1 - linear, 2 - quadratic");
+
+		gd.addNumericField("Minimal standard deviation for poly 1",  this.photo_std_1, 5,7,"",
+				"Minimal standard deviation to use polynomial order 1.");
+		gd.addNumericField("Minimal standard deviation for poly 2",  this.photo_std_2, 5,7,"",
+				"Minimal standard deviation to use polynomial order 2.");
+		gd.addNumericField("Set photo offset (0-keep, 1-balance, 2-set)",this.photo_offs_set, 0,3,"",
+				"0 - keep weighted average offset, 1 - set offset to balance result image, 2 - use next value.");
+		gd.addNumericField("Target value to weighted average offset",this.photo_offs, 5,7,"",
+				"Target weighted (by scales) average offset.");
 		gd.addCheckbox ("Debug photometric calibration",             this.photo_debug,
 				"Generate debug images an text output.");
 		
-		
-		
-		
 		gd.addTab         ("3D", "3D reconstruction");
 		gd.addCheckbox    ("Show generated textures",                                                                this.show_textures);
 		gd.addCheckbox    ("show intermediate results of filtering",                                                 this.debug_filters);
@@ -4251,6 +4270,12 @@ public class CLTParameters {
 		this.photo_min_strength =       gd.getNextNumber();
 		this.photo_max_diff =           gd.getNextNumber();
 		this.photo_order =        (int) gd.getNextNumber();
+		this.photo_std_1 =              gd.getNextNumber();
+		this.photo_std_2 =              gd.getNextNumber();
+
+		this.photo_offs_set =     (int) gd.getNextNumber();
+		this.photo_offs =               gd.getNextNumber();
+		
 		this.photo_debug =              gd.getNextBoolean();
 		
 		this.show_textures=         gd.getNextBoolean();

src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java

@@ -195,6 +195,7 @@ public class IntersceneMatchParameters {
 	public  double  max_search_rms =                1.5;   // good - 0.34, so-so - 0.999
 	public  double  maybe_fom =                     1.0;   // good - 38, second good - 4.5
 	public  double  sure_fom =                     12.0;   // good - 38, second good - 4.5
+	public  boolean treat_serch_fpn =              false;  // use FPN (higher) thresholds during search (even if offset is not small)
 	
 	
 	// Reference scene disparity 
@@ -627,6 +628,8 @@ public class IntersceneMatchParameters {
 				"Minimal acceptable Figure of Merit (semi-total strength divided by standard deviation of offsets), will look for the best among matching.");
 		gd.addNumericField("\"Sure\" FOM",                           this.sure_fom, 5,7,"",
 				"Definitely sufficient FOM (semi-total strength divided by standard deviation of offsets), will non continue looking for better.");
+		gd.addCheckbox ("Treat search all FPN",                      this.treat_serch_fpn,
+				"Use FPN (higher) thresholds during search (even if offset is not small).");
 		
 		
 		gd.addMessage  ("Reference scene disparity");
@@ -972,6 +975,7 @@ public class IntersceneMatchParameters {
 		this.max_search_rms =           gd.getNextNumber();
 		this.maybe_fom =                gd.getNextNumber();
 		this.sure_fom =                 gd.getNextNumber();
+		this.treat_serch_fpn =          gd.getNextBoolean();
 		this.use_combo_dsi =            gd.getNextBoolean();
 		this.use_lma_dsi =              gd.getNextBoolean();
 		this.fpn_remove =               gd.getNextBoolean();
@@ -1252,6 +1256,7 @@ public class IntersceneMatchParameters {
 		properties.setProperty(prefix+"max_search_rms",       this.max_search_rms+"");      // double
 		properties.setProperty(prefix+"maybe_fom",            this.maybe_fom+"");           // double
 		properties.setProperty(prefix+"sure_fom",             this.sure_fom+"");            // double
+		properties.setProperty(prefix+"treat_serch_fpn",        this.treat_serch_fpn+"");   // boolean
 		properties.setProperty(prefix+"use_combo_dsi",        this.use_combo_dsi+"");       // boolean
 		properties.setProperty(prefix+"use_lma_dsi",          this.use_lma_dsi+"");         // boolean
 		properties.setProperty(prefix+"fpn_remove",           this.fpn_remove+"");          // boolean
@@ -1485,6 +1490,7 @@ public class IntersceneMatchParameters {
 		if (properties.getProperty(prefix+"max_search_rms")!=null)       this.max_search_rms=Double.parseDouble(properties.getProperty(prefix+"max_search_rms"));
 		if (properties.getProperty(prefix+"maybe_fom")!=null)            this.maybe_fom=Double.parseDouble(properties.getProperty(prefix+"maybe_fom"));
 		if (properties.getProperty(prefix+"sure_fom")!=null)             this.sure_fom=Double.parseDouble(properties.getProperty(prefix+"sure_fom"));
+		if (properties.getProperty(prefix+"treat_serch_fpn")!=null)      this.treat_serch_fpn=Boolean.parseBoolean(properties.getProperty(prefix+"treat_serch_fpn"));		
 		if (properties.getProperty(prefix+"use_combo_dsi")!=null)        this.use_combo_dsi=Boolean.parseBoolean(properties.getProperty(prefix+"use_combo_dsi"));		
 		if (properties.getProperty(prefix+"use_lma_dsi")!=null)          this.use_lma_dsi=Boolean.parseBoolean(properties.getProperty(prefix+"use_lma_dsi"));
 		if (properties.getProperty(prefix+"fpn_remove")!=null)           this.fpn_remove=Boolean.parseBoolean(properties.getProperty(prefix+"fpn_remove"));
@@ -1735,6 +1741,7 @@ public class IntersceneMatchParameters {
 		imp.max_search_rms =        this.max_search_rms;
 		imp.maybe_fom =             this.maybe_fom;
 		imp.sure_fom =              this.sure_fom;
+		imp.treat_serch_fpn       = this.treat_serch_fpn;
 		imp.use_combo_dsi         = this.use_combo_dsi;
 		imp.use_lma_dsi           = this.use_lma_dsi;
 		imp.fpn_remove            = this.fpn_remove;

src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java

@@ -2452,7 +2452,11 @@ public class QuadCLT extends QuadCLTCPU {
 			final QuadCLT     ref_scene, // now - may be null - for testing if scene is rotated ref
 			final double      min_strength,
 			final double      max_diff,     //  30.0
-			final int         photo_order, // 0 - offset only, 1 - linear, 2 - quadratic
+			int               photo_order,  //   0 - offset only, 1 - linear, 2 - quadratic
+			final double      photo_std_1,  //  50.0;   // Minimal standard deviation of the filtered values for poly order 1
+			final double      photo_std_2,	// 200.0;   // Minimal standard deviation of the filtered values for poly order 2
+			final int         photo_offs_set, // 0;     // 0 - keep weighted offset average, 1 - balance result image, 2 - set weighted average to specific value
+			final double      photo_offs,     // 21946; // weighted average offset target value, if photo_offs_set (and not photo_offs_balance)
 			final int         num_refines, // 2
 			final double [][] combo_dsn_final,     // double [][]    combo_dsn_final, // dls,			
 			int               threadsMax,
@@ -2461,7 +2465,7 @@ public class QuadCLT extends QuadCLTCPU {
 		
 		// filter disparity by LMA only, same bg and fg
 			double []         disparity_ref= combo_dsn_final[OpticalFlow.COMBO_DSN_INDX_DISP].clone();
-			for (int i = 00; i < disparity_ref.length; i++) {
+			for (int i = 0; i < disparity_ref.length; i++) {
 				if (combo_dsn_final[OpticalFlow.COMBO_DSN_INDX_STRENGTH][i] < min_strength) {
 					disparity_ref[i] = Double.NaN;
 				} else if (combo_dsn_final[OpticalFlow.COMBO_DSN_INDX_DISP_BG_ALL][i] !=
@@ -2512,13 +2516,10 @@ public class QuadCLT extends QuadCLTCPU {
 			double [] offs_old =    ref_scene.getLwirOffsets();
 			double [] scales_old =  ref_scene.getLwirScales();
 			double [] scales2_old = ref_scene.getLwirScales2();
+			// Calculate avg_pix - average (for all 16 channels)  pixel value and initial good_pix
+			// next passes avg_pix will be modified and more bad pixels added
 			
-			double [] offs_new =    new double [num_sens];
-			double [] scales_new =  new double [num_sens];
-			double [] scales2_new = new double [num_sens];
-			
-			for (int nref = 0; nref < num_refines; nref++) {
-				Arrays.fill(avg_pix, 0.0);
+			double avg_img = 0.0;
 			int num_good = 0;
 			for (int i = 0; i < len; i++) {
 				for (int n = 0; n < num_sens; n++) {
@@ -2527,15 +2528,33 @@ public class QuadCLT extends QuadCLTCPU {
 				}
 				avg_pix[i] /= dpixels.length;
 				if (good_pix[i]) {
+					avg_img += avg_pix[i];
 					num_good++;
 				}
 			}
-				double [][][] pa_data = new double [num_sens][num_good][2];
+			avg_img /= num_good;
+			double wavg_offs = photo_offs;
+			boolean set_offs = true;
+			if (photo_offs_set == 1) { // balance
+				for (int i = 0; i < len; i++) {
+					avg_pix[i] -= avg_img;
+				}
+				set_offs = false;
+			} else if (photo_offs_set == 0) {
+				double s = 1.0;
+				wavg_offs = 0.0;
+				for (int n = 0; n < num_sens; n++) {
+					s *= scales_old[n];
+					wavg_offs += offs_old[n]*scales_old[n];
+				}
+				s = Math.pow(s, 1.0/num_sens);
+				wavg_offs /= num_sens * s;
+			}
+			
+			// recover original "raw" pixel values undoing old corrections
 			double [][] raw = new double [num_sens][len];
 			for (int nsens = 0; nsens < num_sens; nsens++) {
 				Arrays.fill(raw[nsens],Double.NaN); // debug only
-					
-					int indx = 0;
 				for (int i = 0; i < len; i++) if (good_pix[i]){
 					double A0 = scales2_old[nsens];
 					double B0 = scales_old[nsens];
@@ -2545,10 +2564,63 @@ public class QuadCLT extends QuadCLTCPU {
 					double c = (C0*C0*A0 -C0 * B0 - dpixels[nsens][i]);
 					double p = -c/b;
 					if (Math.abs(a) >= min_abs_a) {
-							p= (-b + Math.sqrt(b*b - 4 * a * c))/(2 * a);
+						double d2 = b*b - 4 * a * c;
+						if (d2 < 0) {
+							System.out.println("calibratePhotometric2() 0: Failed quadratic: a = "+a+", b = "+b+", c="+c);
+							return false;
+						}
+						p= (-b + Math.sqrt(d2))/(2 * a);
 					}
 					raw[nsens][i] = p;
-						pa_data[nsens][indx][0] = p; // dpixels[nsens][i]; 
+				}
+            }
+			
+			double [] offs_new =    new double [num_sens];
+			double [] scales_new =  new double [num_sens];
+			double [] scales2_new = new double [num_sens];
+			double [] avg_chn =     new double [num_sens];
+			double [] std_chn =     new double [num_sens];
+			double std = 0.0;
+			for (int nref = 0; nref < num_refines; nref++) {
+				Arrays.fill(avg_chn, 0.0);
+				Arrays.fill(std_chn, 0.0);
+				num_good = 0;
+				for (int i = 0; i < len; i++) {
+					if (good_pix[i]) {
+						num_good++;
+						for (int n = 0; n < num_sens; n++) {
+							avg_chn[n] += dpixels[n][i];
+							std_chn[n] += dpixels[n][i] * dpixels[n][i];
+						}
+					}
+				}
+				std = 0.0;
+				for (int n = 0; n < num_sens; n++) {
+					avg_chn[n] /= num_good;
+					std_chn[n] /= num_good;
+					std_chn[n] = Math.sqrt(std_chn[n] - avg_chn[n] * avg_chn[n]);
+					std += std_chn[n] * std_chn[n];
+				}
+				std = Math.sqrt(std/num_sens);
+				if ((photo_order > 0) && (std < photo_std_1)) {
+					photo_order = 0;
+					System.out.println ("Standard deviation = "+std+" < "+photo_std_1+", adjusting only offsets. nref = "+nref);
+					nref = -1;
+					continue; // restart
+				} else if ((photo_order > 1) && (std < photo_std_2)) {
+					photo_order = 1;
+					System.out.println ("Standard deviation = "+std+" < "+photo_std_2+", adjusting only offsets and scales. nref = "+nref);
+					nref = -1;
+					continue; // restart
+				} else if (debug) {
+					System.out.println ("Standard deviation = "+std+", nref = "+nref);
+				}
+				
+				double [][][] pa_data = new double [num_sens][num_good][2];
+				for (int nsens = 0; nsens < num_sens; nsens++) {
+					int indx = 0;
+					for (int i = 0; i < len; i++) if (good_pix[i]){
+						pa_data[nsens][indx][0] = raw[nsens][i]; // dpixels[nsens][i]; 
 						pa_data[nsens][indx][1] = avg_pix[i];
 						indx++;
 					}
@@ -2573,7 +2645,12 @@ public class QuadCLT extends QuadCLTCPU {
 					double A = a;
 					double C = -c/b;
 					if (Math.abs(a) >= min_abs_a) {
-						C = (-b + Math.sqrt(b*b - 4*a*c))/(2 * a);
+						double d2 = b*b - 4*a*c;
+						if (d2 < 0) {
+							System.out.println("calibratePhotometric2() 1: Failed quadratic: a = "+a+", b = "+b+", c="+c);
+							return false;
+						}
+						C = (-b + Math.sqrt(d2))/(2 * a);
 					}
 					double B = 2 * C * a + b;
 					scales2_new[nsens] = A;
@@ -2594,6 +2671,14 @@ public class QuadCLT extends QuadCLTCPU {
 							pa_data[nsens][i][1] /= scales_avg;
 						}
 					}
+					for (int i = 0; i < avg_pix.length; i++) { // for the next iteration
+						avg_pix[i] /= scales_avg;
+					}
+					//avg_pix[i]
+					if (debug) {
+						System.out.println ("scales_avg = "+scales_avg+", nref = "+nref);
+					}
+
 				}				
 				// make scales2 be average zero, re-run scales
 				if (photo_order > 1) {
@@ -2606,10 +2691,15 @@ public class QuadCLT extends QuadCLTCPU {
 						scales2_new[nsens] -= scales2_offset;
 					}
 					// modify pa_data, re-run linear
+					
 					for (int nsens = 0; nsens < num_sens; nsens++) {
-						for (int i = 0; i < num_good; i++) {
-							double poffs = pa_data[nsens][i][0] - offs_new[nsens];
-							pa_data[nsens][i][1] -=poffs*poffs*scales2_new[nsens];
+						int indx = 0;
+//						for (int i = 0; i < num_good; i++) {
+						for (int i = 0; i < avg_pix.length; i++) if (good_pix[i]){
+							double poffs = pa_data[nsens][indx][0] - offs_new[nsens];
+							pa_data[nsens][indx][1] -= poffs*poffs*scales2_new[nsens];
+//							avg_pix[i] -= poffs*poffs*scales2_offset/num_sens; // for the next iteration
+							indx++;
 						}
 						pa_coeff[nsens] =(new PolynomialApproximation(0)).polynomialApproximation1d(
 								pa_data[nsens], 1);
@@ -2618,7 +2708,48 @@ public class QuadCLT extends QuadCLTCPU {
 						scales_new [nsens] = b;
 						offs_new[nsens] =   -c/b;
 					}
+					// re-normalize scales_new
+					double scales_avg = 1.0;
+					for (int nsens = 0; nsens < num_sens; nsens++) {
+						scales_avg *= scales_new [nsens];
 					}
+					scales_avg = Math.pow(scales_avg, 1.0/num_sens);
+					for (int nsens = 0; nsens < num_sens; nsens++) {
+						scales_new [nsens] /= scales_avg;
+						scales2_new [nsens] /= scales_avg;
+						for (int i = 0; i < num_good; i++) {
+							pa_data[nsens][i][1] /= scales_avg;
+						}
+					}
+					for (int i = 0; i < avg_pix.length; i++) { // for the next iteration
+						avg_pix[i] /= scales_avg;
+					}
+					//avg_pix[i]
+					if (debug) {
+						System.out.println ("normalization after quadratic: scales_avg = "+scales_avg+", nref = "+nref);
+					}
+				}
+				
+				// update offsets if set_offs
+				if (set_offs) {
+					double wa = 0.0;
+					for (int nsens = 0; nsens < num_sens; nsens++) {
+						wa += offs_new[nsens] * scales_new [nsens];
+					}
+					wa /= num_sens;
+					double a = wavg_offs - wa;
+					for (int nsens = 0; nsens < num_sens; nsens++) {
+						offs_new[nsens] += a/scales_new [nsens];
+					}
+					for (int i = 0; i < avg_pix.length; i++) if (good_pix[i]){
+						avg_pix[i] -= a;
+					}
+					if (debug) {
+						System.out.println ("calibratePhotometric2(): wa = "+wa+", a="+a);
+					}
+				}
+				
+				
 				
 				if (true) { // debug) { during debug,
 					System.out.println("DEBUG: calibratePhotometric() nref="+nref);

src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java

@@ -8798,6 +8798,10 @@ if (debugLevel > -100) return true; // temporarily !
 		    		}
 		    		double pts_scale = clt_parameters.imp.video_fps/clt_parameters.imp.sensor_fps;
 		    		String shellCommand;
+		    		//ffmpeg -i input_file.mkv -c copy -metadata:s:v:0 stereo_mode=1 output_file.mkv 
+		    		//https://ffmpeg.org/ffmpeg-formats.html
+		    		//ffmpeg -i sample_left_right_clip.mpg -an -c:v libvpx -metadata stereo_mode=left_right -y stereo_clip.webm
+		    		//anaglyph_cyan_red 
 		    		if (this_stereo_width > 0) {//  add padding to stereo video
 		    			int padded_width= 16* ( (int) Math.round((this_stereo_width + stereo_gap) * stereo_phone_width/stereo_intereye/32));
 		    			shellCommand = String.format(