Before Java 1.8

src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java

@@ -702,6 +702,7 @@ private Panel panel1,
 			addButton("Inter Test",                 panelClt5, color_stop);
 			addButton("Inter Pairs",                panelClt5, color_process);
 			addButton("Inter LMA",                  panelClt5, color_stop);
+			addButton("Inter Series",               panelClt5, color_process);
 			plugInFrame.add(panelClt5);
 		}
 
@@ -5116,6 +5117,14 @@ private Panel panel1,
         CLT_PARAMETERS.batch_run = true;
         interPairsLMA();
     	return;
+/* ======================================================================== */
+    } else if (label.equals("Inter Series")) {
+        DEBUG_LEVEL=MASTER_DEBUG_LEVEL;
+    	EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL);
+        CLT_PARAMETERS.batch_run = true;
+        interSeriesLMA();
+    	return;
+    	
 /* ======================================================================== */
     } else if (label.equals("Inter LMA")) {
         DEBUG_LEVEL=MASTER_DEBUG_LEVEL;
@@ -6636,6 +6645,77 @@ private Panel panel1,
 	}
 	
 	
+	public boolean interSeriesLMA() {
+		long startTime=System.nanoTime();
+		// load needed sensor and kernels files
+		if (!prepareRigImages()) return false;
+		String configPath=getSaveCongigPath();
+		if (configPath.equals("ABORT")) return false;
+		setAllProperties(PROPERTIES); // batchRig may save properties with the model. Extrinsics will be updated, others should be set here
+		if (DEBUG_LEVEL > -2){
+			System.out.println("++++++++++++++ Testing Interscene processing ++++++++++++++");
+		}
+		
+		if (CLT_PARAMETERS.useGPU()) { // only init GPU instances if it is used
+			if (GPU_TILE_PROCESSOR == null) {
+				try {
+					GPU_TILE_PROCESSOR = new GPUTileProcessor(CORRECTION_PARAMETERS.tile_processor_gpu);
+				} catch (Exception e) {
+					System.out.println("Failed to initialize GPU class");
+					// TODO Auto-generated catch block
+					e.printStackTrace();
+					return false;
+				} //final int        debugLevel);
+			}
+			if (CLT_PARAMETERS.useGPU(false) && (QUAD_CLT != null) && (GPU_QUAD == null)) { // if GPU main is needed
+				try {
+					GPU_QUAD = GPU_TILE_PROCESSOR.new GpuQuad(
+							QUAD_CLT,
+							4,
+							3);
+				} catch (Exception e) {
+					System.out.println("Failed to initialize GpuQuad class");
+					// TODO Auto-generated catch block
+					e.printStackTrace();
+					return false;
+				} //final int        debugLevel);
+				QUAD_CLT.setGPU(GPU_QUAD);
+			}
+		}
+		
+		try {
+			TWO_QUAD_CLT.interSeriesLMA(
+					QUAD_CLT, // QuadCLT quadCLT_main,
+//					QUAD_CLT_AUX, // QuadCLT quadCLT_aux,
+					CLT_PARAMETERS,  // EyesisCorrectionParameters.DCTParameters           dct_parameters,
+					DEBAYER_PARAMETERS, //EyesisCorrectionParameters.DebayerParameters     debayerParameters,
+					COLOR_PROC_PARAMETERS, //EyesisCorrectionParameters.ColorProcParameters colorProcParameters,
+					COLOR_PROC_PARAMETERS_AUX, //EyesisCorrectionParameters.ColorProcParameters colorProcParameters_aux,
+					CHANNEL_GAINS_PARAMETERS, //CorrectionColorProc.ColorGainsParameters     channelGainParameters,
+					RGB_PARAMETERS, //EyesisCorrectionParameters.RGBParameters             rgbParameters,
+					EQUIRECTANGULAR_PARAMETERS, // EyesisCorrectionParameters.EquirectangularParameters equirectangularParameters,
+					PROPERTIES,  // Properties                                           properties,
+					true, // false, // boolean                                              reset_from_extrinsics,
+					THREADS_MAX, //final int          threadsMax,  // maximal number of threads to launch
+					UPDATE_STATUS, //final boolean    updateStatus,
+					DEBUG_LEVEL);
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} //final int        debugLevel);
+		if (configPath!=null) {
+			saveTimestampedProperties( // save config again
+					configPath,      // full path or null
+					null, // use as default directory if path==null
+					true,
+					PROPERTIES);
+		}
+		System.out.println("batchRig(): Processing finished at "+
+				IJ.d2s(0.000000001*(System.nanoTime()-startTime),3)+" sec, --- Free memory="+
+				Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")");
+		return true;
+	}
+	
 	
 	
 

src/main/java/com/elphel/imagej/tileprocessor/ErsCorrection.java

@@ -232,7 +232,20 @@ public class ErsCorrection extends GeometryCorrection {
 	public double [] getErsATR_d2t() {
 		return ers_watr_center_d2t;
 	}
-	
+
+	public void setErsDt(
+			double []    ers_xyz_dt,
+			double []    ers_atr_dt) {
+		this.ers_wxyz_center_dt = ers_xyz_dt;
+		this.ers_watr_center_dt = ers_atr_dt;
+	}
+	public void setErsD2t(
+			double []    ers_xyz_d2t,
+			double []    ers_atr_d2t) {
+		this.ers_wxyz_center_d2t = ers_xyz_d2t;
+		this.ers_watr_center_d2t = ers_atr_d2t;
+	}
+
 	
 	
 	
@@ -286,7 +299,7 @@ public class ErsCorrection extends GeometryCorrection {
 		String prefix = parent_prefix+SCENES_PREFIX+"_"; 
 		for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements();) {
 			String key = (String) e.nextElement();
-			if (key.startsWith(prefix)) {
+			if (key.startsWith(prefix) && !key.endsWith("t")) { // _dt, _d2t
 				timestamps.add(key.substring(prefix.length()));
 			}
 		}
@@ -899,15 +912,6 @@ public class ErsCorrection extends GeometryCorrection {
 			if (xyzw == null) {
 				return null;
 			}
-			if (xyzw[3] == 0.0) { // infinity
-			/*
-				if (xyzw[2] > 0) {
-					for (int i = 0; i < 3; i++) {
-						xyzw[i] = -xyzw[i];	
-					}
-				}
-				*/
-			}
 			if (xyzw[2] > 0) {
 				return null; // can not match object behind the camera
 			}
@@ -1914,6 +1918,23 @@ public class ErsCorrection extends GeometryCorrection {
 		}
 	}
 	
+	public static double [][] combineXYZATR(
+			double [] reference_xyz,
+			double [] reference_atr,
+			double [] scene_xyz,
+			double [] scene_atr)
+	{
+		Rotation   ref_rotation=   new Rotation(RotationOrder.YXZ, ROT_CONV, reference_atr[0],reference_atr[1],reference_atr[2]);
+		Rotation   scene_rotation= new Rotation(RotationOrder.YXZ, ROT_CONV, scene_atr[0],    scene_atr[1],    scene_atr[2]);
+		Vector3D   ref_offset =    new Vector3D(reference_xyz);
+		Vector3D   scene_offset =  new Vector3D(scene_xyz);
+		Vector3D   offset = ref_offset.add(ref_rotation.applyTo(scene_offset));
+		Rotation   rotation = ref_rotation.applyTo(scene_rotation);
+		double []  angles = rotation.getAngles(RotationOrder.YXZ, ROT_CONV);
+		double []  xyz = offset.toArray();
+		return new double[][] {xyz,angles};
+		
+	}
 	
 	public double [] getImageCoordinatesERS(
 			double [] xyzw,
@@ -2155,6 +2176,9 @@ public class ErsCorrection extends GeometryCorrection {
 		double[][] derivatives = new double[DP_NUM_PARS+1][]; // includes [0] - pXpYD vector
 		// scene pX, pY, Disparity
 		derivatives[0] = pXpYD_scene;
+		if (Double.isNaN(pXpYD_scene[0]) || Double.isNaN(pXpYD_scene[1]) || Double.isNaN(pXpYD_scene[2])) {
+			return null;
+		}
 		// derivatives by the reference parameters, starting with /dpX, /dpY, /dd
 		for (int indx = DW_DPX; indx <= DW_DZ; indx++) {
 			int vindx = indx+1;
@@ -2167,6 +2191,9 @@ public class ErsCorrection extends GeometryCorrection {
 			} else {
 				derivatives[vindx] = matrixTimesVector(dpscene_dxyz, reference_vectors[vindx]).toArray();
 			}
+			if (Double.isNaN(derivatives[vindx][0]) || Double.isNaN(derivatives[vindx][1]) || Double.isNaN(derivatives[vindx][2])) {
+				return null;
+			}
 		}
 		for (int indx = DP_DSVAZ; indx < DP_NUM_PARS; indx++) { // 15,16, ...
 			int indx_out = indx+1; // 16, 17,
@@ -2216,7 +2243,7 @@ public class ErsCorrection extends GeometryCorrection {
 		double pYcd = pXpYD[1] - 0.5 * this.pixelCorrectionHeight;
 		double r_scale = 0.001 * this.pixelSize / this.distortionRadius;
 		double rDn =    Math.sqrt(pXcd*pXcd + pYcd*pYcd) * r_scale; // relative to distortion radius
-		double rND2RD = correctDistortions?(getRByRDist(rDn, false)): 1.0;
+		double rND2RD = correctDistortions?(getRByRDist(rDn, false)): 1.0; // NaN
 		// rD - in mm
 //		double rD = rDn * this.distortionRadius; // Math.sqrt(pXcd*pXcd + pYcd*pYcd)*0.001*this.pixelSize; // distorted radius in a virtual center camera
 		// pXc, pYc - in pixels

src/main/java/com/elphel/imagej/tileprocessor/GeometryCorrection.java

@@ -4403,13 +4403,15 @@ matrix([[-0.125, -0.125,  0.125,  0.125, -0.125,  0.125, -0.   , -0.   ,   -0.
 		}
 		if (rDist < 0) { // not used in lwir
 			if (debug)System.out.println("getRByRDist("+IJ.d2s(rDist,3)+"): rDist < 0");
-			return Double.NaN;
+			rDist=0.0; // Double.NaN;
 		}
 		double findex = rDist/this.stepR;
 		int index=(int) Math.floor(findex);
 		if (index>=(this.rByRDist.length-2)) { // not used in lwir
 			if (debug) System.out.println("getRByRDist("+IJ.d2s(rDist,3)+"): index="+index+">="+(this.rByRDist.length-2));
-			return Double.NaN;
+			// happens for 3D points far offcenter, should be filtered out by FoV
+			return this.rByRDist.length-1;
+//			return Double.NaN;
 		}
 		double mu = findex - index;
 		double mu2 = mu * mu;

src/main/java/com/elphel/imagej/tileprocessor/IntersceneLma.java

@@ -176,7 +176,7 @@ public class IntersceneLma {
 				scenesCLT[1],      // final QuadCLT     scene_QuadClt,
 				scenesCLT[0],      // final QuadCLT     reference_QuadClt,
 				debug_level);      // final int         debug_level)
-		double [][] wjtj = getWJtJlambda( // USED in lwir
+		double [][] wjtj = getWJtJlambda( // USED in lwir all NAN
 				0.0,               // final double      lambda,
 				last_jt);               // final double [][] jt)
 		for (int i = 0; i < parameters_vector.length; i++) {
@@ -252,7 +252,7 @@ public class IntersceneLma {
 			}
 
 		} else { // improved over initial ?
-			if (last_rms[0] < initial_rms[0]) {
+			if (last_rms[0] < initial_rms[0]) { // NaN
 				rslt[0] = true;
 				if (debug_level > 0) System.out.println("Step "+iter+": Failed to converge, but result improved over initial");
 			} else {
@@ -270,6 +270,16 @@ public class IntersceneLma {
 				}
 			}
 		}
+		if ((debug_level > -2) && !rslt[0]) { // failed
+			if ((debug_level > 1) || (iter == 1) || last_run) {
+				System.out.println("LMA failed on iteration = "+iter);
+				String [] lines = printOldNew(true); // boolean allvectors)
+				for (String line : lines) {
+					System.out.println(line);
+				}
+			}
+			System.out.println();
+		}
 
 		return rslt[0]? iter : -1;
 	}
@@ -487,6 +497,9 @@ public class IntersceneLma {
 		double full_weight = asum_weight.sum();
 		pure_weight = s_pure/full_weight;
 		final double s = 1.0/asum_weight.sum();
+		if (Double.isNaN(s)) {
+			System.out.println("normalizeWeights(): s == NaN");
+		}
 		ai.set(0);
 		for (int ithread = 0; ithread < threads.length; ithread++) {
 			threads[ithread] = new Thread() {
@@ -578,13 +591,16 @@ public class IntersceneLma {
 									scene_rot_matrix,            // Matrix    scene_rot_matrix,        // single rotation (direct) matrix for the scene
 									debug_level);                // int debug_level);
 							if (deriv_params!= null) {
-								fx[2 * iMTile + 0] = deriv_params[0][0]; // pX
-								fx[2 * iMTile + 1] = deriv_params[0][1]; // pY
-								if (jt != null) {
-									for (int i = 0; i < par_indices.length; i++) {
-										int indx = par_indices[i] + 1;
-										jt[i][2 * iMTile + 0] = deriv_params[indx][0]; // pX
-										jt[i][2 * iMTile + 1] = deriv_params[indx][1]; // pY (disparity is not used)
+								boolean bad_tile = false;
+								if (!bad_tile) {
+									fx[2 * iMTile + 0] = deriv_params[0][0]; // pX
+									fx[2 * iMTile + 1] = deriv_params[0][1]; // pY
+									if (jt != null) {
+										for (int i = 0; i < par_indices.length; i++) {
+											int indx = par_indices[i] + 1;
+											jt[i][2 * iMTile + 0] = deriv_params[indx][0]; // pX
+											jt[i][2 * iMTile + 1] = deriv_params[indx][1]; // pY (disparity is not used)
+										}
 									}
 								}
 							}

src/main/java/com/elphel/imagej/tileprocessor/OpticalFlowParameters.java

@@ -72,6 +72,7 @@ public class OpticalFlowParameters {
 	
 	public int         debug_level_optical =  1;
 	public int         debug_level_iterate = -1;
+	public boolean     enable_debug_images =  true;
 	
 	public void dialogQuestions(GenericJTabbedDialog gd) {
 		gd.addMessage("Intraframe ERS to pose");
@@ -159,7 +160,8 @@ public class OpticalFlowParameters {
 		
 		gd.addNumericField("Debug level correlation iterations",                              this.debug_level_iterate,  0,4,"",
 				"Apply during Optical Flow refinement itgerations");
-		
+	    gd.addCheckbox    ("Enable debug images",  this.enable_debug_images,
+			"When false - no debug images will be generated regardless of debug level");
 	}	
 	
 	public void dialogAnswers(GenericJTabbedDialog gd) {
@@ -202,6 +204,7 @@ public class OpticalFlowParameters {
 		this.test_corr_rad_max =                (int) gd.getNextNumber();
 		this.debug_level_optical =    (int) gd.getNextNumber();
 		this.debug_level_iterate =    (int) gd.getNextNumber();
+		this.enable_debug_images =          gd.getNextBoolean();
 		
 	}	
 	
@@ -243,7 +246,7 @@ public class OpticalFlowParameters {
 		properties.setProperty(prefix+"test_corr_rad_max",        this.test_corr_rad_max+"");
 		properties.setProperty(prefix+"debug_level_optical",      this.debug_level_optical+"");
 		properties.setProperty(prefix+"debug_level_iterate",      this.debug_level_iterate+"");
-	
+		properties.setProperty(prefix+"enable_debug_images",      this.enable_debug_images+"");
 	}	
 	
 	public void getProperties(String prefix,Properties properties){
@@ -285,7 +288,8 @@ public class OpticalFlowParameters {
 		if (properties.getProperty(prefix+"test_corr_rad_max")!=null)        this.test_corr_rad_max=Integer.parseInt(properties.getProperty(prefix+"test_corr_rad_max"));
 		if (properties.getProperty(prefix+"debug_level_optical")!=null)      this.debug_level_optical=Integer.parseInt(properties.getProperty(prefix+"debug_level_optical"));
 		if (properties.getProperty(prefix+"debug_level_iterate")!=null)      this.debug_level_iterate=Integer.parseInt(properties.getProperty(prefix+"debug_level_iterate"));
-
+		if (properties.getProperty(prefix+"enable_debug_images")!=null)      this.enable_debug_images=Boolean.parseBoolean(properties.getProperty(prefix+"enable_debug_images"));
+		
 	}
 	@Override
 	public OpticalFlowParameters clone() throws CloneNotSupportedException {
@@ -325,6 +329,7 @@ public class OpticalFlowParameters {
 		ofp.test_corr_rad_max =             this.test_corr_rad_max;
 		ofp.debug_level_optical =           this.debug_level_optical;
 		ofp.debug_level_iterate =           this.debug_level_iterate;
+		ofp.enable_debug_images =           this.enable_debug_images;
 		return ofp;
 		
 	}	

src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java

@@ -8391,7 +8391,6 @@ if (debugLevel > -100) return true; // temporarily !
 
 	}
 	
-
 	public void TestInterLMA(
 			QuadCLT                                              quadCLT_main, // tiles should be set
 			CLTParameters             clt_parameters,
@@ -8484,6 +8483,72 @@ if (debugLevel > -100) return true; // temporarily !
 		System.out.println("End of test");
 
 
+	}
+
+	public void interSeriesLMA(
+			QuadCLT                                              quadCLT_main, // tiles should be set
+			CLTParameters             clt_parameters,
+			EyesisCorrectionParameters.DebayerParameters         debayerParameters,
+			ColorProcParameters                                  colorProcParameters,
+			ColorProcParameters                                  colorProcParameters_aux,
+			CorrectionColorProc.ColorGainsParameters             channelGainParameters,
+			EyesisCorrectionParameters.RGBParameters             rgbParameters,
+			EyesisCorrectionParameters.EquirectangularParameters equirectangularParameters,
+			Properties                                           properties,
+			boolean                                              reset_from_extrinsics,
+			final int        threadsMax,  // maximal number of threads to launch
+			final boolean    updateStatus,
+			final int        debugLevel)  throws Exception
+	{
+		if ((quadCLT_main != null) && (quadCLT_main.getGPU() != null)) {
+			quadCLT_main.getGPU().resetGeometryCorrection();
+			quadCLT_main.gpuResetCorrVector(); // .getGPU().resetGeometryCorrectionVector();
+		}
+		// final boolean    batch_mode = clt_parameters.batch_run;
+		this.startTime=System.nanoTime();
+		String [] sourceFiles0=quadCLT_main.correctionsParameters.getSourcePaths();
+		QuadCLT.SetChannels [] set_channels_main = quadCLT_main.setChannels(debugLevel);
+		if ((set_channels_main == null) || (set_channels_main.length==0)) {
+			System.out.println("No files to process (of "+sourceFiles0.length+")");
+			return;
+		}
+		QuadCLT.SetChannels [] set_channels=quadCLT_main.setChannels(debugLevel);
+		
+		QuadCLT [] quadCLTs = new QuadCLT [set_channels.length]; 
+		for (int i = 0; i < quadCLTs.length; i++) {
+			quadCLTs[i] = quadCLT_main.spawnQuadCLT(
+					set_channels[i].set_name,
+					clt_parameters,
+					colorProcParameters, //
+					threadsMax,
+					debugLevel);
+			// temporarily fix wrong sign:
+			ErsCorrection ers = (ErsCorrection) (quadCLTs[i].getGeometryCorrection());
+//			if (reset_from_extrinsics) {
+//				System.out.println("Reset ERS parameters from intraframe extrinsics");
+//				ers.setupERSfromExtrinsics();
+//			}
+			quadCLTs[i].setDSRBG(
+					clt_parameters, // CLTParameters  clt_parameters,
+					threadsMax,     // int            threadsMax,  // maximal number of threads to launch
+					updateStatus,   // boolean        updateStatus,
+					debugLevel);    // int            debugLevel)
+///			quadCLTs[i].showDSIMain();
+		}
+		
+		
+		OpticalFlow opticalFlow = new OpticalFlow(
+				threadsMax, // int            threadsMax,  // maximal number of threads to launch
+				updateStatus); // boolean        updateStatus);
+		
+		opticalFlow.adjustSeries(
+				clt_parameters, // CLTParameters  clt_parameters,			
+				clt_parameters.ofp.k_prev, // k_prev,
+				quadCLTs, // QuadCLT [] scenes, // ordered by increasing timestamps
+				clt_parameters.ofp.debug_level_optical); // 1); // -1); // int debug_level);
+		System.out.println("End of interSeriesLMA()");
+
+
 	}
 	
 	public void batchLwirRig(