Implemented combining of the interscene 2D correlations amon tile and (bd27e7b0) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java

+185 −8

Original line number	Diff line number	Diff line
		@@ -1358,16 +1358,33 @@ public class ImageDtt extends ImageDttCPU {
		final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
		final double half_disparity, // 5.0; // Reduce weight twice for this disparity
		final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice

		final boolean neibs_nofpn_only, // consolidate neighbors fot non-fpn tiles only!
		final boolean redo_both, // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
		final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
		final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
		final double scale_neibs_td, // scale threshold for the transform-domain average maximums
		final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones

		final int debug_tileX,
		final int debug_tileY,
		final int threadsMax, // maximal number of threads to launch
		final int globalDebugLevel)
		{

		{ /*
		boolean use_neibs = true; // false; // true;
		final boolean neibs_nofpn_only = false; // consolidate neighbors fot non-fpn tiles only!
		final double scale_neibs_pd = use_neibs? 0.5 : 0;
		final double scale_neibs_td = use_neibs? 0.5 : 0;
		final double scale_avg_weight = 0.5; // reduce influence of the averaged correlations compared to the single-tile ones
		final int min_num_neibs = 4; // plus center, total number >= (min_num_neibs+1)
		final boolean redo_both = true; // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
		*/
		if (this.gpuQuad == null) {
		System.out.println("clt_process_tl_interscene(): this.gpuQuad is null, bailing out");
		return null;
		}
		//boolean debugTile0 =(tileX == debug_tileX) && (tileY == debug_tileY) && (globalDebugLevel > 2); // 0);

		// final int min_neibs = clt_parameters.imp.min_neibs;
		final boolean extra_sum = true; // use sum of pixel-domain correlations (TD have artifacts for low contrast
		// - maybe -related to float vs. double - not tested yet
		@@ -1456,11 +1473,29 @@ public class ImageDtt extends ImageDttCPU {

		final Thread[] threads = newThreadArray(threadsMax);
		final AtomicInteger ai = new AtomicInteger(0);
		final int [] map_corr_indices=new int[tilesX*tilesY];
		Arrays.fill(map_corr_indices, -1);
		// create indices for neighbors
		for (int ithread = 0; ithread < threads.length; ithread++) {
		threads[ithread] = new Thread() {
		@Override
		public void run() {
		for (int iCorrTile = ai.getAndIncrement(); iCorrTile < num_tiles; iCorrTile = ai.getAndIncrement()) {
		int nTile = (corr_indices[iCorrTile* used_sensors_list.length] >> GPUTileProcessor.CORR_NTILE_SHIFT);
		map_corr_indices[nTile] = iCorrTile;
		}
		}
		};
		}
		startAndJoin(threads);
		ai.set(0);

		for (int ithread = 0; ithread < threads.length; ithread++) {
		threads[ithread] = new Thread() {
		@Override
		public void run() {
		int tileY,tileX,nTile; // , chn;
		TileNeibs tn = new TileNeibs(tilesX,tilesY);
		for (int iCorrTile = ai.getAndIncrement(); iCorrTile < num_tiles; iCorrTile = ai.getAndIncrement()) {
		nTile = (corr_indices[iCorrTile* used_sensors_list.length] >> GPUTileProcessor.CORR_NTILE_SHIFT);
		tileY = nTile / tilesX;
		@@ -1473,6 +1508,7 @@ public class ImageDtt extends ImageDttCPU {
		boolean [] fpn_mask = null;
		double min_str = min_str_nofpn; // higher threshold when FPN is possible
		double min_str_sum = min_str_sum_nofpn; // higher threshold when FPN is possible
		boolean is_fpn=false;
		if ((fpn_offsets != null) && (fpn_offsets[nTile] != null)) {
		double fpn_x = transform_size - 1 - fpn_offsets[nTile][0]; // 0 -> 7.0
		double fpn_y = transform_size - 1 - fpn_offsets[nTile][1]; // 0 -> 7.0
		@@ -1485,12 +1521,13 @@ public class ImageDtt extends ImageDttCPU {
		for (int iy = min_y; iy <= max_y; iy++) {
		for (int ix = min_x; ix <= max_x; ix++) {
		int indx = iy * corr_size + ix;
		fcorr2D[fcorr2D_indx][indx] = 0;
		// fcorr2D[fcorr2D_indx][indx] = 0;
		fpn_mask[indx] = true;
		}
		}
		min_str = min_str_fpn;
		min_str_sum = min_str_sum_fpn;
		is_fpn = true;
		}
		double [][] corrs = new double [corrs_len + extra_len][];
		// copy correlation tiles from the GPU's floating point arrays
		@@ -1520,8 +1557,13 @@ public class ImageDtt extends ImageDttCPU {
		}
		}
		}
		if (is_fpn) {
		for (int i = 0; i < corr_length; i++) if (fpn_mask[i]){
		corrs[corrs_len - 1][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
		}
		}

		if (dcorr_tiles != null) { // This will be visualized
		if (dcorr_tiles != null) { // This will be visualized (only for visualization?)
		int index_es = getNumSensors() + extra_len;
		dcorr_tiles[iCorrTile] = new double[getNumSensors()+1 + extra_len][];
		if (extra_sum) {
		@@ -1544,6 +1586,11 @@ public class ImageDtt extends ImageDttCPU {
		dcorr_tiles[iCorrTile][abs_sens][i] = gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		if (is_fpn) {
		for (int i = 0; i < corr_length; i++) if (fpn_mask[i]){
		dcorr_tiles[iCorrTile][used_sensors_list.length-1][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
		}
		}
		}
		// final double [][][][] motion_vectors, // [tilesY][tilesX][][] -> [][][num_sel_sensors+1][2]
		if (motion_vectors != null) { // TODO: now used only as debug, may be removed later
		@@ -1568,8 +1615,10 @@ public class ImageDtt extends ImageDttCPU {
		// now calculate only for composite
		double [] mv_pd = new double [3];
		double [] mv_td = new double [3];
		boolean retry_pd=false, retry_td=false;
		boolean neib_en = !(is_fpn && neibs_nofpn_only);
		if (pd_weight > 0.0) {
		mv_pd = Correlation2d.getMaxXYCm(
		mv_pd = Correlation2d.getMaxXYCm( // last, average
		corrs[corrs.length-1], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		centroid_radius, // double radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
		@@ -1579,14 +1628,18 @@ public class ImageDtt extends ImageDttCPU {
		false); // boolean debug)
		if (mv_pd != null) {
		if (mv_pd[2] < min_str) {
		if (neib_en && (scale_neibs_pd > 0)) {
		retry_pd = mv_pd[2] >= min_str * scale_neibs_pd;
		}
		mv_pd = null;
		} else {
		mv_pd[2] -= min_str;
		// mv_pd[2] -= min_str;
		mv_pd[2] -= min_str * scale_neibs_pd;
		}
		}
		}
		if (td_weight > 0.0) {
		mv_td = Correlation2d.getMaxXYCm(
		mv_td = Correlation2d.getMaxXYCm( // pre-last - sharp (in FD)
		corrs[corrs.length-2], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		centroid_radius, // double radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
		@@ -1596,12 +1649,136 @@ public class ImageDtt extends ImageDttCPU {
		false); // boolean debug)
		if (mv_td != null) {
		if (mv_td[2] < min_str_sum) {
		if (neib_en && (scale_neibs_td > 0)) {
		retry_td = mv_td[2] >= min_str_sum * scale_neibs_td;
		}
		mv_td = null;
		} else {
		mv_td[2] -= min_str_sum;
		// mv_td[2] -= min_str_sum;
		mv_td[2] -= min_str_sum * scale_neibs_td;
		}
		}
		}
		// calculate averages from neighbors
		// will replace corrs[] with averages
		if (retry_pd \|\| retry_td) {
		if (redo_both) {
		retry_pd \|= retry_td; // here could be just true
		retry_td \|= retry_pd; // here could be just true
		}
		int num_neibs = 1; // center tile itself
		// calculate num_neibs first not to bother if there are too few of them if (num_neibs > )
		for (int dir = 0; dir < tn.numNeibs(); dir++) {
		int nTile1 = tn.getNeibIndex(nTile, dir);
		if ((nTile1 >= 0) && (map_corr_indices[nTile1] >= 0)) {
		num_neibs++;
		}
		}
		scale = 1.0/getNumSensors();
		if (num_neibs > min_num_neibs) {
		for (int dir = 0; dir < tn.numNeibs(); dir++) {
		int nTile1 = tn.getNeibIndex(nTile, dir);
		if ((nTile1 >= 0) && (map_corr_indices[nTile1] >= 0)) {
		int iCorrTile1 = map_corr_indices[nTile1];
		for (int isens = corrs_len - 1; isens >= 0; isens--) {
		int nsens = used_sensors_list.length - corrs_len + isens;
		int fcorr2D_indx = iCorrTile1 * used_sensors_list.length + nsens;
		if (fcorr2D_indx < 0) {
		System.out.println("BUG");
		}
		for (int i = 0; i < corr_length; i++) { // java.lang.ArrayIndexOutOfBoundsException: Index -1 out of bounds for length 69802
		corrs[isens][i] += gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		// direcly accumulating, without preservation of per-sensor data
		if (!use_partial && extra_sum) {
		for (int nsens = 0; nsens < (used_sensors_list.length - 1); nsens++) {
		int fcorr2D_indx = iCorrTile1 * used_sensors_list.length + nsens;
		for (int i = 0; i < corr_length; i++) {
		corrs[corrs_len][i] += scale * gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		}
		}
		}
		if (use_partial && extra_sum) {
		corrs[corrs_len] = new double [corr_length];
		for (int isens = corrs_len - 2; isens >= 0; isens--) {
		for (int i = 0; i < corr_length; i++) {
		corrs[corrs_len][i] += scale*corrs[isens][i];
		}
		}
		}
		// divide by number of neighbors to get averages
		double scale_neibs = 1.0/num_neibs;
		for (int isens=0; isens < corrs.length; isens++) {
		for (int i = 0; i < corr_length; i++) {
		corrs[isens][i] *= scale_neibs;
		}
		}
		// zero fpn mask for sum slice
		if (is_fpn) {
		for (int i = 0; i < corr_length; i++) if (fpn_mask[i]){
		corrs[corrs_len - 1][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
		}
		}

		// update dcorr_tiles for visualization
		// just copy last corrs to last dcorr_tiles[iCorrTile]
		if (dcorr_tiles != null) { // This will be visualized (only for visualization?)
		int num_dslices = Math.min(corrs.length,dcorr_tiles[iCorrTile].length);
		for (int islice = 0; islice < num_dslices; islice++) {
		System.arraycopy(
		corrs[corrs.length-num_dslices+islice],
		0,
		dcorr_tiles[iCorrTile][dcorr_tiles[iCorrTile].length-num_dslices+islice],
		0,
		corrs[corrs.length-num_dslices+islice].length);
		}
		}
		// recalculate motion vectors mv_pd, mv_td with now averaged data
		// final double scale_avg_weight = 0.5; // reduce influence of the averaged correlations compared to the single-tile ones

		if (retry_pd && (pd_weight > 0.0)) {
		mv_pd = Correlation2d.getMaxXYCm( // last, average
		corrs[corrs.length-1], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		centroid_radius, // double radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
		n_recenter, // int refine, // re-center window around new maximum. 0 -no refines (single-pass)
		null, // boolean [] fpn_mask,
		false, // boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
		false); // boolean debug)
		if (mv_pd != null) {
		if (mv_pd[2] < (min_str * scale_neibs_pd)) {
		mv_pd = null;
		} else {
		mv_pd[2] -= min_str * scale_neibs_pd;
		mv_pd[2] *= scale_avg_weight;
		}
		}
		}
		// TODO: do the same with td
		if (retry_td && (td_weight > 0.0)) {
		mv_td = Correlation2d.getMaxXYCm( // pre-last - sharp (in FD)
		corrs[corrs.length-2], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		centroid_radius, // double radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
		n_recenter, // int refine, // re-center window around new maximum. 0 -no refines (single-pass)
		fpn_mask, // boolean [] fpn_mask,
		false, // boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
		false); // boolean debug)
		if (mv_td != null) {
		if (mv_td[2] < (min_str_sum * scale_neibs_td)) {
		mv_td = null;
		} else {
		mv_td[2] -= min_str_sum * scale_neibs_td;
		mv_td[2] *= scale_avg_weight;
		}
		}
		}
		} // if (num_neibs > min_num_neibs) {
		}

		if ((mv_td != null) \|\| (mv_pd != null)) {
		double [] mv = new double[3 + (use3D? 2 :0)]; // keep for disparity/strength
		if (mv_pd != null) {

src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java

+63 −0

Original line number	Diff line number	Diff line
		@@ -226,6 +226,15 @@ public class IntersceneMatchParameters {
		public double pd_weight = 0.5; // correlations (post) accumulated in PD
		public boolean td_nopd_only = false; // true; // only use TD accumulated data if no safe PD is available for the tile.

		// averaging with neighbors for interscene matching
		public boolean use_neibs = true; // false; // true;
		public boolean neibs_nofpn_only = false; // consolidate neighbors fot non-fpn tiles only!
		public boolean redo_both = true; // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
		public int min_num_neibs = 4; // plus center, total number >= (min_num_neibs+1)
		public double scale_neibs_pd = 0.5; // scale threshold for the pixel-domain average maximums
		public double scale_neibs_td = 0.5; // scale threshold for the transform-domain average maximums
		public double scale_avg_weight = 0.5; // reduce influence of the averaged correlations compared to the single-tile ones

		public double min_str_fpn = 0.2; // 0.25; // minimal correlation strength for all but TD-accumulated layer
		public double min_str_sum_fpn = 0.5; // 0.8; // minimal correlation strength for TD-accumulated layer

		@@ -739,6 +748,24 @@ public class IntersceneMatchParameters {
		"Mix argmax from PD-accumulated correlation.");
		gd.addCheckbox ("TD when no PD only", this.td_nopd_only,
		"Use argmax from TD only if PD data is not available for this tile.");

		gd.addMessage ("Averaging inter-scene 2D correlations with 8 immediate neighbors");
		gd.addCheckbox ("Use averaging neighbors", this.use_neibs,
		"Average tile 2D correlation with 8 immediate neighbors if single-tile correlation is not strong enough.");
		// TODO: Consider/Implement pure TD averaging
		gd.addCheckbox ("No-FPN averaging only", this.neibs_nofpn_only,
		"Use averaging with neighbors only if tile offset is large enough not to have FPN interference.");
		gd.addCheckbox ("Use averaging for both PD and TD", this.redo_both,
		"Recalculate tile average for both TD and PD if at least one of them is not strong enough.");
		gd.addNumericField("Minimal neighbors to average", this.min_num_neibs, 0,3,"",
		"Minimal number of defined (non-null) neighbors 2D correlations (of 8) to use averaging.");
		gd.addNumericField("Scale PD thershold", this.scale_neibs_pd, 5,7,"",
		"Use averaging for tiles that are stronger than scaled threshold for a single-tile motion vector for pixel-domain values.");
		gd.addNumericField("Scale TD threshold", this.scale_neibs_td, 5,7,"",
		"Use averaging for tiles that are stronger than scaled threshold for a single-tile motion vector for transform-domain values.");
		gd.addNumericField("Scale averaged strengths", this.scale_avg_weight, 5,7,"",
		"Scale averaged motion vector strengths (for each of the TD and PD).");

		gd.addMessage ("Filtering motion vectors");
		gd.addNumericField("Minimal correlation strength (non-sum)", this.min_str, 5,7,"",
		"Minimal correlation strength for individual correlation and for pixel-domain averaged one. Weeker tiles results are removed.");
		@@ -1166,6 +1193,15 @@ public class IntersceneMatchParameters {
		this.td_weight = gd.getNextNumber();
		this.pd_weight = gd.getNextNumber();
		this.td_nopd_only = gd.getNextBoolean();

		this.use_neibs = gd.getNextBoolean();
		this.neibs_nofpn_only = gd.getNextBoolean();
		this.redo_both = gd.getNextBoolean();
		this.min_num_neibs = (int) gd.getNextNumber();
		this.scale_neibs_pd = gd.getNextNumber();
		this.scale_neibs_td = gd.getNextNumber();
		this.scale_avg_weight = gd.getNextNumber();

		this.min_str = gd.getNextNumber();
		this.min_str_fpn = gd.getNextNumber();
		this.min_str_sum = gd.getNextNumber();
		@@ -1505,6 +1541,15 @@ public class IntersceneMatchParameters {
		properties.setProperty(prefix+"td_weight", this.td_weight+""); // double
		properties.setProperty(prefix+"pd_weight", this.pd_weight+""); // double
		properties.setProperty(prefix+"td_nopd_only", this.td_nopd_only+""); // boolean

		properties.setProperty(prefix+"use_neibs", this.use_neibs+""); // boolean
		properties.setProperty(prefix+"neibs_nofpn_only", this.neibs_nofpn_only+""); // boolean
		properties.setProperty(prefix+"redo_both", this.redo_both+""); // boolean
		properties.setProperty(prefix+"min_num_neibs", this.min_num_neibs+""); // int
		properties.setProperty(prefix+"scale_neibs_pd", this.scale_neibs_pd+""); // double
		properties.setProperty(prefix+"scale_neibs_td", this.scale_neibs_td+""); // double
		properties.setProperty(prefix+"scale_avg_weight", this.scale_avg_weight+""); // double

		properties.setProperty(prefix+"min_str", this.min_str+""); // double
		properties.setProperty(prefix+"min_str_fpn", this.min_str_fpn+""); // double
		properties.setProperty(prefix+"min_str_sum", this.min_str_sum+""); // double
		@@ -1799,6 +1844,15 @@ public class IntersceneMatchParameters {
		if (properties.getProperty(prefix+"td_weight")!=null) this.td_weight=Double.parseDouble(properties.getProperty(prefix+"td_weight"));
		if (properties.getProperty(prefix+"pd_weight")!=null) this.pd_weight=Double.parseDouble(properties.getProperty(prefix+"pd_weight"));
		if (properties.getProperty(prefix+"td_nopd_only")!=null) this.td_nopd_only=Boolean.parseBoolean(properties.getProperty(prefix+"td_nopd_only"));

		if (properties.getProperty(prefix+"use_neibs")!=null) this.use_neibs=Boolean.parseBoolean(properties.getProperty(prefix+ "use_neibs"));
		if (properties.getProperty(prefix+"neibs_nofpn_only")!=null) this.neibs_nofpn_only=Boolean.parseBoolean(properties.getProperty(prefix+"neibs_nofpn_only"));
		if (properties.getProperty(prefix+"redo_both")!=null) this.redo_both=Boolean.parseBoolean(properties.getProperty(prefix+ "redo_both"));
		if (properties.getProperty(prefix+"min_num_neibs")!=null) this.min_num_neibs=Integer.parseInt(properties.getProperty(prefix+ "min_num_neibs"));
		if (properties.getProperty(prefix+"scale_neibs_pd")!=null) this.scale_neibs_pd=Double.parseDouble(properties.getProperty(prefix+ "scale_neibs_pd"));
		if (properties.getProperty(prefix+"scale_neibs_td")!=null) this.scale_neibs_td=Double.parseDouble(properties.getProperty(prefix+ "scale_neibs_td"));
		if (properties.getProperty(prefix+"scale_avg_weight")!=null) this.scale_avg_weight =Double.parseDouble(properties.getProperty(prefix+ "scale_avg_weight"));

		if (properties.getProperty(prefix+"min_str")!=null) this.min_str=Double.parseDouble(properties.getProperty(prefix+"min_str"));
		if (properties.getProperty(prefix+"min_str_fpn")!=null) this.min_str_fpn=Double.parseDouble(properties.getProperty(prefix+"min_str_fpn"));
		if (properties.getProperty(prefix+"min_str_sum")!=null) this.min_str_sum=Double.parseDouble(properties.getProperty(prefix+"min_str_sum"));
		@@ -2104,6 +2158,15 @@ public class IntersceneMatchParameters {
		imp.td_weight = this.td_weight;
		imp.pd_weight = this.pd_weight;
		imp.td_nopd_only = this.td_nopd_only;

		imp.use_neibs = this.use_neibs;
		imp.neibs_nofpn_only = this.neibs_nofpn_only;
		imp.redo_both = this.redo_both;
		imp.min_num_neibs = this.min_num_neibs;
		imp.scale_neibs_pd = this.scale_neibs_pd;
		imp.scale_neibs_td = this.scale_neibs_td;
		imp.scale_avg_weight = this.scale_avg_weight;

		imp.min_str = this.min_str;
		imp.min_str_fpn = this.min_str_fpn;
		imp.min_str_sum = this.min_str_sum;

src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java

+34 −6

File changed.

Preview size limit exceeded, changes collapsed.