disabling SfM if there is not much movement (bc3b706a) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java

+15 −2

Original line number	Diff line number	Diff line
		@@ -44,6 +44,9 @@ public class IntersceneMatchParameters {

		public boolean sfm_use = true; // use SfM to improve depth map
		public double sfm_min_base = 2.0; // use SfM if baseline exceeds this
		public double sfm_min_gain = 5.0; // Minimal SfM gain to apply SfM to the depth map
		public double sfm_min_frac = 0.5; // Minimal fraction of defined tiles to have SfM correction

		public int sfm_num_pairs = 16; // desired number of SfM pairs to average
		public double sfp_tolerance = 0.05; // average SfM pairs if their baselines differ less
		public int sfm_readjust = 5; // number of SfM readjustment cycles
		@@ -461,8 +464,10 @@ public class IntersceneMatchParameters {
		"Use SfM for the depth map enhancement for laterally moving camera.");
		gd.addNumericField("Minimal SfM baseline", this.sfm_min_base, 5,8,"m",
		"Use SfM only if the baseline (lateral offset between scenes in a series) exceeds this value.");


		gd.addNumericField("Minimal SfM gain", this.sfm_min_gain, 5,8,"",
		"Minimal SfM gain to apply SfM to the depth map.");
		gd.addNumericField("Minimal SfM fraction", this.sfm_min_frac, 5,8,"",
		"Minimal fraction of defined tiles to have SfM correction.");
		gd.addNumericField("Number of SfM pairs to average", this.sfm_num_pairs, 0,3,"",
		"Desired number of SfM pairs to average, will be reduced if not enough or tolerance (below) is violated.");
		gd.addNumericField("SfP inter-pair baseline tolerance", this.sfp_tolerance, 5,8,"",
		@@ -1174,6 +1179,8 @@ public class IntersceneMatchParameters {
		this.ims_mount_xyz = IntersceneMatchParameters. StringToDoubles(gd.getNextString(), 3);
		this.sfm_use = gd.getNextBoolean();
		this.sfm_min_base = gd.getNextNumber();
		this.sfm_min_gain = gd.getNextNumber();
		this.sfm_min_frac = gd.getNextNumber();
		this.sfm_num_pairs = (int) gd.getNextNumber();
		this.sfp_tolerance = gd.getNextNumber();
		this.sfm_readjust = (int) gd.getNextNumber();
		@@ -1549,6 +1556,8 @@ public class IntersceneMatchParameters {
		properties.setProperty(prefix+"ims_mount_xyz", IntersceneMatchParameters.doublesToString(this.ims_mount_xyz));
		properties.setProperty(prefix+"sfm_use", this.sfm_use + ""); // boolean
		properties.setProperty(prefix+"sfm_min_base", this.sfm_min_base+""); // double
		properties.setProperty(prefix+"sfm_min_gain", this.sfm_min_gain+""); // double
		properties.setProperty(prefix+"sfm_min_frac", this.sfm_min_frac+""); // double
		properties.setProperty(prefix+"sfm_num_pairs", this.sfm_num_pairs+""); // int
		properties.setProperty(prefix+"sfp_tolerance", this.sfp_tolerance+""); // double
		properties.setProperty(prefix+"sfm_readjust", this.sfm_readjust+""); // int
		@@ -1884,6 +1893,8 @@ public class IntersceneMatchParameters {
		if (properties.getProperty(prefix+"ims_mount_xyz")!=null) this.ims_mount_xyz= IntersceneMatchParameters.StringToDoubles(properties.getProperty(prefix+"ims_mount_xyz"),3);
		if (properties.getProperty(prefix+"sfm_use")!=null) this.sfm_use=Boolean.parseBoolean(properties.getProperty(prefix+"sfm_use"));
		if (properties.getProperty(prefix+"sfm_min_base")!=null) this.sfm_min_base=Double.parseDouble(properties.getProperty(prefix+"sfm_min_base"));
		if (properties.getProperty(prefix+"sfm_min_gain")!=null) this.sfm_min_gain=Double.parseDouble(properties.getProperty(prefix+"sfm_min_gain"));
		if (properties.getProperty(prefix+"sfm_min_frac")!=null) this.sfm_min_frac=Double.parseDouble(properties.getProperty(prefix+"sfm_min_frac"));
		if (properties.getProperty(prefix+"sfm_num_pairs")!=null) this.sfm_num_pairs=Integer.parseInt(properties.getProperty(prefix+"sfm_num_pairs"));
		if (properties.getProperty(prefix+"sfp_tolerance")!=null) this.sfp_tolerance=Double.parseDouble(properties.getProperty(prefix+"sfp_tolerance"));
		if (properties.getProperty(prefix+"sfm_readjust")!=null) this.sfm_readjust=Integer.parseInt(properties.getProperty(prefix+"sfm_readjust"));
		@@ -2246,6 +2257,8 @@ public class IntersceneMatchParameters {
		imp.ims_mount_xyz = this.ims_mount_xyz.clone();
		imp.sfm_use = this.sfm_use;
		imp.sfm_min_base = this.sfm_min_base;
		imp.sfm_min_gain = this.sfm_min_gain;
		imp.sfm_min_frac = this.sfm_min_frac;
		imp.sfm_num_pairs = this.sfm_num_pairs;
		imp.sfp_tolerance = this.sfp_tolerance;
		imp.sfm_readjust = this.sfm_readjust;

src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java

+49 −1

Original line number	Diff line number	Diff line
		@@ -4775,8 +4775,54 @@ public class OpticalFlow {
		// 1. Reference scene DSI
		// while ((quadCLTs[ref_index] == null) \|\| (quadCLTs[ref_index].getBlueSky() == null)) { // null
		// String center_ts = null; // name of the center scene - not yet known
		double [][] combo_dsn_final = null;
		while ((quadCLTs[last_index] == null) \|\| !quadCLTs[last_index].hasBlueSky()) { // null
		//hasBlueSky()
		// creating early photometric

		/*
		if (photo_en && !reuse_video) {
		if (debugLevel > -3) {
		System.out.println("** Running photometric equalization ***");
		}
		if (combo_dsn_final == null) { // always re-read?
		/// combo_dsn_final =quadCLTs[ref_index].readDoubleArrayFromModelDirectory( // always re-read?
		/// "-INTER-INTRA-LMA", // String suffix,
		/// 0, // int num_slices, // (0 - all)
		/// null); // int [] wh);
		combo_dsn_final =quadCLTs[ref_index].restoreComboDSI(true); // also sets quadCLTs[ref_index].dsi and blue sky
		}

		double [][] combo_dsn_final_filtered =
		conditionComboDsnFinal(
		true, // boolean use_conf, // use configuration parameters, false - use following
		clt_parameters, // CLTParameters clt_parameters,
		combo_dsn_final, // double [][] combo_dsn_final, // dls,
		quadCLTs[ref_index], // QuadCLT scene,
		debugLevel); // int debugLevel);// > 0
		// replace
		runPhotometric(
		clt_parameters, // CLTParameters clt_parameters,
		colorProcParameters, // ColorProcParameters colorProcParameters,
		set_channels[ref_index].set_name, // String set_name
		quadCLTs, // QuadCLT[] quadCLTs,
		earliest_scene, // int earliest_scene,
		last_index, // int last_scene,
		ref_index, // int ref_index,
		combo_dsn_final_filtered, // double [][] combo_dsn_final_filtered,
		threadsMax, // int threadsMax, // maximal number of threads to launch
		updateStatus, // boolean updateStatus,
		debugLevel); // int debugLevel);

		} else {
		if (debugLevel> -3) {
		System.out.println("Using photometric calibration from scene "+quadCLTs[ref_index].getPhotometricScene());
		}
		}

		*/


		if (build_ref_dsi) {
		TwoQuadCLT.copyJP4src( // actually there is no sense to process multiple image sets. Combine with other
		// processing?
		@@ -5017,7 +5063,7 @@ public class OpticalFlow {
		// just in case that orientations were calculated before:
		// earliest_scene = getEarliestScene(quadCLTs);

		double [][] combo_dsn_final = null;
		// double [][] combo_dsn_final = null;
		// below ref_index is not necessary the last (fix all where it is supposed to be the last
		ErsCorrection ers_reference = quadCLTs[ref_index].getErsCorrection();
		while (!reuse_video && ((quadCLTs[ref_index].getNumOrient() < min_num_orient) \|\| (quadCLTs[ref_index].getNumAccum() < min_num_interscene))) {
		@@ -5049,6 +5095,8 @@ public class OpticalFlow {
		}
		}
		QuadCLT[][][] scenes_seq_pairs = new QuadCLT[3][num_avg_pairs][2];

		// TODO: calculate horizontal offset and compare with sfm_min_base
		for (int i = 0; i < num_avg_pairs; i++) {
		scenes_seq_pairs[0][i][0] = quadCLTs[ref_index + num_avg_pairs - 1 - i];
		scenes_seq_pairs[0][i][1] = quadCLTs[earliest_scene + num_avg_pairs - 1 - i];

src/main/java/com/elphel/imagej/tileprocessor/QuadCLTCPU.java

+15 −1

Original line number	Diff line number	Diff line
		@@ -6261,9 +6261,13 @@ public class QuadCLTCPU {
		int [][] fileIndices=new int [numImagesToProcess][2]; // file index, channel number
		int index=0;
		for (int nFile=0;nFile<enabledFiles.length;nFile++){ // enabledFiles not used anymore?
		// if ((single_set_name != null) && (correctionsParameters.getNameFromTiff(sourceFiles[nFile])==null)) {
		// System.out.println("sourceFiles["+nFile+"]==null");
		// continue;
		// }
		if ( (sourceFiles[nFile]!=null) &&
		(sourceFiles[nFile].length()>1) &&
		((single_set_name == null) \|\| (correctionsParameters.getNameFromTiff(sourceFiles[nFile]).contains(single_set_name)))) { // not used in lwir
		((single_set_name == null) \|\| ((correctionsParameters.getNameFromTiff(sourceFiles[nFile]) != null)&& correctionsParameters.getNameFromTiff(sourceFiles[nFile]).contains(single_set_name)))) { // not used in lwir
		int [] channels= fileChannelToSensorChannels(correctionsParameters.getChannelFromSourceTiff(sourceFiles[nFile]));
		if (channels!=null){
		for (int i=0;i<channels.length;i++) if (eyesisCorrections.isChannelEnabled(channels[i])){
		@@ -6690,6 +6694,16 @@ public class QuadCLTCPU {
		pixels[i] += fcorr;
		}
		}
		} else {
		for (int i = 0; i < imp_srcs.length; i++) {
		if (ImagejJp4Tiff.needsFix000E6410C435(imp_srcs[i])) {
		needs_fix = i;
		break; // only one, and always 6
		}
		}
		if (needs_fix >= 0) {
		System.out.println("Need to correct "+imp_srcs[needs_fix].getTitle()+" **************");
		}
		}

		channelLwirApplyEqualize( // now apply (was part of channelLwirEqualize() )

src/main/java/com/elphel/imagej/tileprocessor/sfm/StructureFromMotion.java

+94 −7

Original line number	Diff line number	Diff line
		@@ -250,6 +250,9 @@ public class StructureFromMotion {
		boolean show_2d_correlations = false; // true;
		boolean show_disp_corr = false; //true;
		boolean show_disp_seq = true;
		double sfm_min_gain = 10.0;
		double sfm_min_frac = 0.5;

		int num_readjust = 5;
		double min_strength = 0.4;
		double range_disparity_offset = clt_parameters.imp.range_disparity_offset;
		@@ -438,10 +441,19 @@ public class StructureFromMotion {
		clt_parameters, // final CLTParameters clt_parameters,
		corr2d_PD_use, // final double [][] corr2d_PD,
		dpXYddisp, // final double [][] dpXYddisp,
		sfm_min_gain, // double sfm_min_gain,
		sfm_min_frac, //final double sfm_min_frac,
		centroid_radius, // final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
		n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
		corr_size, //final int corr_size, // 15
		debugLevel); // final int debugLevel);
		if (disp_corr == null) {
		if (debugLevel > -3) {
		System.out.println("sfmPair_debug(): not enough SfM tiles, bailing out");
		}
		return null;
		}

		for (int nTile = 0; nTile< disp_corr.length; nTile++) if (disp_corr[nTile] != null){
		if (disp_corr[nTile][1] > min_strength) {
		ref_disparity[nTile] += disp_corr[nTile][0];
		@@ -511,7 +523,8 @@ public class StructureFromMotion {
		final int debugLevel) {
		boolean show_disp_corr = false; //true;
		boolean show_disp_seq = true;

		final double sfm_min_gain = clt_parameters.imp.sfm_min_gain;
		final double sfm_min_frac = clt_parameters.imp.sfm_min_frac;
		final int num_readjust = clt_parameters.imp.sfm_readjust; // 5;
		final double min_strength1 = clt_parameters.imp.sfm_min_str1; // 0.4; // update if correction strength exceeds
		final double min_strength16= clt_parameters.imp.sfm_min_str16; // 0.4; // update if correction strength exceeds
		@@ -639,6 +652,9 @@ public class StructureFromMotion {
		scene_pairs, // final QuadCLT[][] scene_pairs,
		scenes_xyzatr, // final double[][][][] scenes_xyzatr, // new double[num_pairs][2][][]; // 2 scenes
		scenes_xyzatr_dt, // final double[][][][] scenes_xyzatr_dt, // new double[num_pairs][2][][];// 2 scenes
		sfm_min_gain, // final double sfm_min_gain,
		sfm_min_frac, // final double sfm_min_frac,

		mb_max_gain, // final double mb_max_gain,
		range_disparity_offset, // final double range_disparity_offset, // disparity at actual infinity // clt_parameters.imp.range_disparity_offset ; final double mb_max_gain,
		corr_fz_inter, // final double corr_fz_inter,
		@@ -648,6 +664,13 @@ public class StructureFromMotion {
		n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
		batch_mode, // final boolean batch_mode,
		debugLevel); // final int debugLevel)
		if (sfmCorr == null) {
		if (debugLevel > -3) {
		System.out.println("sfmPair(): not enough SfM tiles, bailing out");
		}
		return null;
		}

		if ((sfm_shrink > 0) \|\| (fade_sigma > 0.0)) {
		shrinkFadeSfmGain(
		sfm_shrink, // final int sfm_shrink,
		@@ -783,6 +806,9 @@ public class StructureFromMotion {


		final int dbg_tile = -( 52+58*80);
		final double sfm_min_base = clt_parameters.imp.sfm_min_base;
		final double sfm_min_gain = clt_parameters.imp.sfm_min_gain;
		final double sfm_min_frac = clt_parameters.imp.sfm_min_frac;
		final int num_readjust = clt_parameters.imp.sfm_readjust; // 5;
		final double min_strength1 = clt_parameters.imp.sfm_min_str1; // 0.4; // update if correction strength exceeds
		final double min_strength16= clt_parameters.imp.sfm_min_str16; // 0.4; // update if correction strength exceeds
		@@ -921,7 +947,28 @@ public class StructureFromMotion {
		}
		}
		}

		// check each pair has sufficient baseline and contains exactly 2 scenes
		for (int nset=0; nset < num_sets; nset++) {
		int num_pairs = scene_pairs_sets[nset].length;
		for (int npair = 0; npair < num_pairs; npair++) {
		double [][][] scenes_xyzatr= scenes_xyzatr_sets[nset][npair];
		if (scenes_xyzatr.length != 2) {
		System.out.println("sfmPairsSet(): BUG: Not a pair of scenes: scenes_xyzatr_sets["+nset+"]["+npair+
		"].length = "+(scenes_xyzatr.length)+" != 2");
		return null;
		}
		double dx = scenes_xyzatr[1][0][0] - scenes_xyzatr[0][0][0];
		double dy = scenes_xyzatr[1][0][1] - scenes_xyzatr[0][0][1];
		double base = Math.sqrt(dxdx + dydy);
		if (base < sfm_min_base) {
		if (debugLevel > -3) {
		System.out.println("sfmPairsSet(): stereo base for nset="+nset+", npair="+npair+
		" = "+base+" < "+sfm_min_base+", bailing out from SfM ranging.");
		}
		return null;
		}
		}
		}

		for (int ntry = 0; ntry < num_readjust; ntry++) {
		final SfmCorr [] sfmCorrCombo = new SfmCorr [tilesX*tilesY];
		@@ -947,6 +994,8 @@ public class StructureFromMotion {
		scene_pairs_sets[nset], // final QuadCLT[][] scene_pairs,
		scenes_xyzatr_sets[nset], // final double[][][][] scenes_xyzatr, // new double[num_pairs][2][][]; // 2 scenes
		scenes_xyzatr_dt_sets[nset],// final double[][][][] scenes_xyzatr_dt, // new double[num_pairs][2][][];// 2 scenes
		sfm_min_gain, // final double sfm_min_gain,
		sfm_min_frac, // final double sfm_min_frac,
		mb_max_gain, // final double mb_max_gain,
		range_disparity_offset, // final double range_disparity_offset, // disparity at actual infinity // clt_parameters.imp.range_disparity_offset ; final double mb_max_gain,
		corr_fz_inter, // final double corr_fz_inter,
		@@ -956,6 +1005,13 @@ public class StructureFromMotion {
		n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
		batch_mode, // final boolean batch_mode,
		debugLevel); // final int debugLevel)
		if (sfmCorr == null) {
		if (debugLevel > -3) {
		System.out.println("sfmPairsSet(): not enough SfM tiles, bailing out");
		}
		return null;
		}

		if ((sfm_shrink > 0) \|\| (fade_sigma > 0.0)) {
		shrinkFadeSfmGain(
		sfm_shrink, // final int sfm_shrink,
		@@ -1472,6 +1528,8 @@ public class StructureFromMotion {
		* @param scene_pairs pairs of scene instances to correlate ([1] to [0])
		* @param scenes_xyzatr pairs of scene poses relative to the reference one
		* @param scenes_xyzatr_dt pairs of scene poses derivatives (for ERS)
		* @param sfm_min_gain minimal SfM gain to apply SfM to the depth map
		* @param sfm_min_frac minimal fraction of defined tiles to have SfM correction
		* @param mb_max_gain motion blur correction maximal gain
		* @param range_disparity_offset disparity at actual infinity. Here can be just 0.
		* @param corr_fz_inter Fat zero correction. Recommended value:
		@@ -1484,7 +1542,8 @@ public class StructureFromMotion {
		* @param debugLevel debug level
		* @return per tile array (sparse) of SfmCorr instances, containing tile's SfM gain,
		* individual disparity correction (to add to the current disparity)/strength and
		* optionally averaged between neighbors correction/strength
		* optionally averaged between neighbors correction/strength. Returns null if there
		* insufficient tiles with high enough disparity gain.
		*/
		public static SfmCorr [] getSfmCorr(
		final CLTParameters clt_parameters,
		@@ -1495,6 +1554,8 @@ public class StructureFromMotion {
		final QuadCLT[][] scene_pairs,
		final double[][][][] scenes_xyzatr, // new double[num_pairs][2][][]; // 2 scenes
		final double[][][][] scenes_xyzatr_dt, // new double[num_pairs][2][][];// 2 scenes
		final double sfm_min_gain,
		final double sfm_min_frac,
		final double mb_max_gain,
		final double range_disparity_offset, // disparity at actual infinity // clt_parameters.imp.range_disparity_offset ; final double mb_max_gain,
		final double corr_fz_inter,
		@@ -1607,10 +1668,18 @@ public class StructureFromMotion {
		clt_parameters, // final CLTParameters clt_parameters,
		data_PD[i], // final double [][] corr2d_PD,
		dpXYddisp_avg, // final double [][] dpXYddisp,
		sfm_min_gain, // final double sfm_min_gain,
		sfm_min_frac, // final double sfm_min_frac,
		centroid_radius, // final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
		n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
		corr_size, //final int corr_size, // 15
		debugLevel); // final int debugLevel);
		if (disp_corr[i] == null) {
		if (debugLevel > -3) {
		System.out.println("getSfmCorr(): not enough SfM tiles, bailing out");
		}
		return null;
		}
		}

		ai.set(0);
		@@ -1717,18 +1786,23 @@ public class StructureFromMotion {
		* pixels in X and Y directions.
		* @param clt_parameters processing parameters
		* @param corr2d_PD per-tile sparse array of the 2D correlation results
		* @param dpXYddisp per-tile sparse array of the X,Y components of teh SfM gain
		* @param dpXYddisp per-tile sparse array of the X,Y components of the SfM gain
		* @param sfm_min_gain minimal SfM gain to apply SfM to the depth map
		* @param sfm_min_frac minimal fraction of defined tiles to have SfM correction
		* @param centroid_radius windowing for centroid argmax: 0 - use all tile, >0 - cosine
		* window around local max
		* @param n_recenter number of refining of the argmax
		* @param corr_size 2d correlation tile size (side of a square, normally 15)
		* @param debugLevel debug level
		* @return per tile sparse array of {disparity_correction, strength} pairs
		* @return per tile sparse array of {disparity_correction, strength} pairs. Will return
		* null if number of sfm tiles is less than sfm_min_frac of defined tiles
		*/
		public static double [][] getSfmDisparityCorrectionStrength(
		final CLTParameters clt_parameters,
		final double [][] corr2d_PD,
		final double [][] dpXYddisp,
		final double sfm_min_gain,
		final double sfm_min_frac,
		final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
		final int n_recenter, // when cosine window, re-center window this many times
		final int corr_size, // 15
		@@ -1736,6 +1810,10 @@ public class StructureFromMotion {
		final double [][] disp_corr = new double [corr2d_PD.length][];
		final Thread[] threads = ImageDtt.newThreadArray(QuadCLT.THREADS_MAX);
		final AtomicInteger ai = new AtomicInteger(0);
		final AtomicInteger anum_defined = new AtomicInteger(0);
		final AtomicInteger anum_sfm = new AtomicInteger(0);
		final double sfm_min_gain2 = sfm_min_gain * sfm_min_gain;

		for (int ithread = 0; ithread < threads.length; ithread++) {
		threads[ithread] = new Thread() {
		public void run() {
		@@ -1751,12 +1829,21 @@ public class StructureFromMotion {
		n_recenter, // int refine, // re-center window around new maximum. 0 -no refines (single-pass)
		inv_v, // double [] direction_XY,
		debugLevel > 2); // boolean debug)
		if (corr_cm != null){
		anum_defined.getAndIncrement();
		if (l2 >= sfm_min_gain2) {
		anum_sfm.getAndIncrement();
		disp_corr[nTile] = corr_cm; // may be null
		}
		}
		}
		}
		};
		}
		ImageDtt.startAndJoin(threads);
		if ((anum_sfm.get() == 0) \|\| (anum_sfm.get() < anum_defined.get() * sfm_min_frac)) {
		return null;
		}
		return disp_corr;
		}

Admin message