TD interscene averaging (e83cb48d) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java

+71 −183

Original line number	Diff line number	Diff line
		@@ -1572,18 +1572,22 @@ public class ImageDtt extends ImageDttCPU {
		final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
		final int n_recenter, // when cosine window, re-center window this many times
		final double td_weight, // mix correlations accumulated in TD with
		final double td_neib_weight, // mix correlations accumulated in TD (neibs)
		final double pd_weight, // correlations (post) accumulated in PD
		final boolean td_nopd_only, // only use TD accumulated data if no safe PD is available for the tile.
		final boolean neib_notd_only, // use neighbors only if individual TD is too weak
		final double min_str_nofpn, // = 0.25;
		final double min_str_sum_nofpn,// = 0.8; // 5;
		final double min_str_neib_nofpn,// = 0.8; // 5;
		final double min_str_fpn, // = 0.25;
		final double min_str_sum_fpn, // = 0.8; // 5;
		final double min_str_neib_fpn, // = 0.8; // 5;
		final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
		final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
		final double half_disparity, // 5.0; // Reduce weight twice for this disparity
		final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice

		final boolean neibs_nofpn_only, // consolidate neighbors fot non-fpn tiles only!
		final boolean neibs_nofpn_only, // consolidate neighbors for non-fpn tiles only!
		final boolean redo_both, // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
		final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
		final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
		@@ -1725,6 +1729,7 @@ public class ImageDtt extends ImageDttCPU {
		boolean [] fpn_mask = null;
		double min_str = min_str_nofpn; // higher threshold when FPN is possible
		double min_str_sum = min_str_sum_nofpn; // higher threshold when FPN is possible
		double min_str_neib = min_str_neib_nofpn; // higher threshold when FPN is possible
		boolean is_fpn=false;
		if ((fpn_offsets != null) && (fpn_offsets[nTile] != null)) {
		double fpn_x = transform_size - 1 - fpn_offsets[nTile][0]; // 0 -> 7.0
		@@ -1743,6 +1748,7 @@ public class ImageDtt extends ImageDttCPU {
		}
		min_str = min_str_fpn;
		min_str_sum = min_str_sum_fpn;
		min_str_neib = min_str_neib_fpn;
		is_fpn = true;
		}

		@@ -1769,13 +1775,6 @@ public class ImageDtt extends ImageDttCPU {
		corrs[corrs_len][i] += scale*corrs[isens][i];
		}
		}
		/*
		if (use_partial && (isens < (corrs_len - 1))) { // not including sum
		for (int i = 0; i < corr_length; i++) {
		corrs[corrs_len][i] += scale*corrs[isens][i];
		}
		}
		*/
		}
		// calculate PD sum of individual sensors correlations if they themselves are not preserved
		if (!use_partial && extra_sum && use_full) {
		@@ -1797,33 +1796,12 @@ public class ImageDtt extends ImageDttCPU {
		}

		if (dcorr_tiles != null) { // This will be visualized (only for visualization?)
		// int index_es = getNumSensors() + extra_len;
		int index_es = used_sensors_list.length; // last, OK if extra_len==0
		//used_sensors_list
		// dcorr_tiles[iCorrTile] = new double[getNumSensors()+1 + extra_len][];
		dcorr_tiles[iCorrTile] = new double[used_sensors_list.length + extra_len][];
		if (extra_sum) {
		dcorr_tiles[iCorrTile][index_es] = new double[corr_length];
		}
		/*
		for (int nsens = 0; nsens < used_sensors_list.length; nsens++) { // all but sum
		int abs_sens = used_sensors_list[nsens]; // should fork for neibs to full (2 elements)
		if (abs_sens >= getNumSensors()) {
		abs_sens = getNumSensors(); // last - sum of all sensors
		}
		if ((abs_sens < getNumSensors()) && extra_sum) {
		int fcorr2D_indx = iCorrTile * used_sensors_list.length + nsens;
		for (int i = 0; i < corr_length; i++) {
		dcorr_tiles[iCorrTile][index_es][i] += scale * gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		dcorr_tiles[iCorrTile][abs_sens] = new double[corr_length];
		int fcorr2D_indx = iCorrTile * used_sensors_list.length + nsens;
		for (int i = 0; i < corr_length; i++) {
		dcorr_tiles[iCorrTile][abs_sens][i] = gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		*/

		for (int nsens = 0; nsens < used_sensors_list.length; nsens++) { // all but sum
		int abs_sens = used_sensors_list[nsens]; // should fork for neibs to full (2 elements)
		@@ -1848,11 +1826,10 @@ public class ImageDtt extends ImageDttCPU {
		}
		}
		}
		// final double [][][][] motion_vectors, // [tilesY][tilesX][][] -> [][][num_sel_sensors+1][2]
		if (motion_vectors != null) { // TODO: now used only as debug, may be removed later
		motion_vectors[nTile] = new double [corrs.length][];
		for (int nsens = 0; nsens < corrs.length; nsens++) { // all 18
		double min_vstr = (nsens == (corrs_len - 1))? min_str_sum : min_str;
		double min_vstr = ((nsens >= used_sensors_list.length) \|\| (used_sensors_list[nsens] < getNumSensors()))? min_str : min_str_sum; // (nsens == (corrs_len - 1))? min_str_sum : min_str;
		motion_vectors[nTile][nsens] = Correlation2d.getMaxXYCm(
		corrs[nsens], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		@@ -1871,7 +1848,8 @@ public class ImageDtt extends ImageDttCPU {
		// now calculate only for composite
		double [] mv_pd = new double [3];
		double [] mv_td = new double [3];
		boolean retry_pd=false, retry_td=false;
		double [] mv_neib = new double [3];
		// boolean retry_pd=false, retry_td=false;
		boolean neib_en = !(is_fpn && neibs_nofpn_only);
		if ((pd_weight > 0.0) && (indx_sum_pd >=0)) {
		mv_pd = Correlation2d.getMaxXYCm( // last, average
		@@ -1884,16 +1862,13 @@ public class ImageDtt extends ImageDttCPU {
		false); // boolean debug)
		if (mv_pd != null) {
		if (mv_pd[2] < min_str) {
		if (neib_en && (scale_neibs_pd > 0)) {
		retry_pd = mv_pd[2] >= min_str * scale_neibs_pd;
		}
		mv_pd = null;
		} else {
		// mv_pd[2] -= min_str;
		mv_pd[2] -= min_str * scale_neibs_pd;
		}
		}
		}

		if (td_weight > 0.0) {
		mv_td = Correlation2d.getMaxXYCm( // pre-last - sharp (in FD)
		corrs[indx_sum_td], // corrs.length-2], // double [] data,
		@@ -1905,147 +1880,58 @@ public class ImageDtt extends ImageDttCPU {
		false); // boolean debug)
		if (mv_td != null) {
		if (mv_td[2] < min_str_sum) {
		if (neib_en && (scale_neibs_td > 0)) {
		retry_td = mv_td[2] >= min_str_sum * scale_neibs_td;
		}
		mv_td = null;
		} else {
		// mv_td[2] -= min_str_sum;
		mv_td[2] -= min_str_sum * scale_neibs_td;
		}
		}
		}
		// calculate averages from neighbors
		// will replace corrs[] with averages
		/*
		if (retry_pd \|\| retry_td) {
		if (redo_both) {
		retry_pd \|= retry_td; // here could be just true
		retry_td \|= retry_pd; // here could be just true
		}
		int num_neibs = 1; // center tile itself
		// calculate num_neibs first not to bother if there are too few of them if (num_neibs > )
		for (int dir = 0; dir < tn.numNeibs(); dir++) {
		int nTile1 = tn.getNeibIndex(nTile, dir);
		if ((nTile1 >= 0) && (map_corr_indices[nTile1] >= 0)) {
		num_neibs++;
		}
		}
		scale = 1.0/getNumSensors();
		if (num_neibs > min_num_neibs) {
		for (int dir = 0; dir < tn.numNeibs(); dir++) {
		int nTile1 = tn.getNeibIndex(nTile, dir);
		if ((nTile1 >= 0) && (map_corr_indices[nTile1] >= 0)) {
		int iCorrTile1 = map_corr_indices[nTile1];
		for (int isens = corrs_len - 1; isens >= 0; isens--) {
		int nsens = used_sensors_list.length - corrs_len + isens;
		int fcorr2D_indx = iCorrTile1 * used_sensors_list.length + nsens;
		if (fcorr2D_indx < 0) {
		System.out.println("BUG");
		}
		for (int i = 0; i < corr_length; i++) { // java.lang.ArrayIndexOutOfBoundsException: Index -1 out of bounds for length 69802
		corrs[isens][i] += gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		// directly accumulating, without preservation of per-sensor data
		if (!use_partial && extra_sum) {
		for (int nsens = 0; nsens < (used_sensors_list.length - 1); nsens++) {
		int fcorr2D_indx = iCorrTile1 * used_sensors_list.length + nsens;
		for (int i = 0; i < corr_length; i++) {
		corrs[corrs_len][i] += scale * gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
		}
		}
		}
		}
		}
		if (use_partial && extra_sum) {
		corrs[corrs_len] = new double [corr_length];
		for (int isens = corrs_len - 2; isens >= 0; isens--) {
		for (int i = 0; i < corr_length; i++) {
		corrs[corrs_len][i] += scale*corrs[isens][i];
		}
		}
		}
		// divide by number of neighbors to get averages
		double scale_neibs = 1.0/num_neibs;
		for (int isens=0; isens < corrs.length; isens++) {
		for (int i = 0; i < corr_length; i++) {
		corrs[isens][i] *= scale_neibs;
		}
		}
		// zero fpn mask for sum slice
		if (is_fpn) {
		for (int i = 0; i < corr_length; i++) if (fpn_mask[i]){
		corrs[corrs_len - 1][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
		}
		}

		// update dcorr_tiles for visualization
		// just copy last corrs to last dcorr_tiles[iCorrTile]
		if (dcorr_tiles != null) { // This will be visualized (only for visualization?)
		int num_dslices = Math.min(corrs.length,dcorr_tiles[iCorrTile].length);
		for (int islice = 0; islice < num_dslices; islice++) {
		System.arraycopy(
		corrs[corrs.length-num_dslices+islice],
		0,
		dcorr_tiles[iCorrTile][dcorr_tiles[iCorrTile].length-num_dslices+islice],
		0,
		corrs[corrs.length-num_dslices+islice].length);
		}
		}
		// recalculate motion vectors mv_pd, mv_td with now averaged data
		// final double scale_avg_weight = 0.5; // reduce influence of the averaged correlations compared to the single-tile ones

		if (retry_pd && (pd_weight > 0.0)) {
		mv_pd = Correlation2d.getMaxXYCm( // last, average
		corrs[corrs.length-1], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		centroid_radius, // double radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
		n_recenter, // int refine, // re-center window around new maximum. 0 -no refines (single-pass)
		null, // boolean [] fpn_mask,
		false, // boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
		false); // boolean debug)
		if (mv_pd != null) {
		if (mv_pd[2] < (min_str * scale_neibs_pd)) {
		mv_pd = null;
		} else {
		mv_pd[2] -= min_str * scale_neibs_pd;
		mv_pd[2] *= scale_avg_weight;
		}
		}
		}
		// TODO: do the same with td
		if (retry_td && (td_weight > 0.0)) {
		mv_td = Correlation2d.getMaxXYCm( // pre-last - sharp (in FD)
		corrs[corrs.length-2], // double [] data,
		if ((td_neib_weight > 0.0) && (indx_sum_td_neib>=0) && (!neib_notd_only \|\| (mv_td == null))) {
		mv_neib = Correlation2d.getMaxXYCm( // pre-last - sharp (in FD)
		corrs[indx_sum_td_neib], // corrs.length-2], // double [] data,
		corr_size, // int data_width, // = 2 * transform_size - 1;
		centroid_radius, // double radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
		n_recenter, // int refine, // re-center window around new maximum. 0 -no refines (single-pass)
		fpn_mask, // boolean [] fpn_mask,
		false, // boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
		false); // boolean debug)
		if (mv_td != null) {
		if (mv_td[2] < (min_str_sum * scale_neibs_td)) {
		mv_td = null;
		if (mv_neib != null) {
		if (mv_neib[2] < min_str_neib) {
		mv_neib = null;
		} else {
		mv_td[2] -= min_str_sum * scale_neibs_td;
		mv_td[2] *= scale_avg_weight;
		mv_neib[2] -= min_str_neib * scale_neibs_td;
		}
		}
		}
		} // if (num_neibs > min_num_neibs) {


		if ((mv_td != null) \|\| (mv_pd != null) \|\| (mv_neib != null)) {
		double w_pd = (mv_pd != null) ? pd_weight : 0.0;
		double w_td = (mv_td != null) ? td_weight : 0.0;
		double w_ntd = (mv_neib != null) ? td_neib_weight: 0.0;
		if (td_nopd_only && (w_pd > 0)) {
		w_td = 0.0;
		}
		*/
		if ((mv_td != null) \|\| (mv_pd != null)) {
		if (neib_notd_only && (w_td > 0)) {
		w_ntd = 0.0;
		}
		double sw = w_pd + w_td + w_ntd;
		if (sw > 0) { // should always be > 0 ?
		w_pd /= sw;
		w_td /= sw;
		w_ntd /= sw;
		double [] mv = new double[3 + (use3D? 2 :0)]; // keep for disparity/strength
		if (mv_pd != null) {
		if (mv_pd != null) for (int i = 0; i < mv_pd.length; i++) mv[i] += w_pd * mv_pd[i];
		if (mv_td != null) for (int i = 0; i < mv_td.length; i++) mv[i] += w_td * mv_td[i];
		if (mv_neib != null) for (int i = 0; i < mv_neib.length; i++) mv[i] += w_ntd * mv_neib[i];
		/*
		/// mv = mv_pd;
		System.arraycopy(mv_pd, 0, mv, 0, 3); // keep [3] for optional disparities
		// mv[2] *= pd_weight;
		if ((mv_td != null) && !td_nopd_only) { // mix
		mv[0] = (mv[0] * pd_weight + mv_td[0] * td_weight)/ (pd_weight + td_weight);
		mv[1] = (mv[1] * pd_weight + mv_td[1] * td_weight)/ (pd_weight + td_weight);
		//mv[2] += mv_td[2] * td_weight;
		mv[2] = (mv[2] * pd_weight + mv_td[2] * td_weight)/ (pd_weight + td_weight);
		} // mix
		} else { // (mv_pd == null) && (mv_td != null) below
		@@ -2053,6 +1939,7 @@ public class ImageDtt extends ImageDttCPU {
		System.arraycopy(mv_td, 0, mv, 0, 3); // keep [3] for optional disparities
		//mv[2] *= td_weight;
		}
		*/
		if (mv != null) {
		if (pXpYD == null) {
		coord_motion[0][nTile] = mv;
		@@ -2066,6 +1953,7 @@ public class ImageDtt extends ImageDttCPU {
		}
		}
		}
		}
		};
		}
		startAndJoin(threads);

src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java

+41 −8

File changed.

Preview size limit exceeded, changes collapsed.

src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java

+11 −6

Original line number	Diff line number	Diff line
		@@ -13146,6 +13146,7 @@ public class OpticalFlow {
		//all_fpn
		double min_str = all_fpn ? clt_parameters.imp.min_str_fpn : clt_parameters.imp.min_str;
		double min_str_sum = all_fpn ? clt_parameters.imp.min_str_sum_fpn : clt_parameters.imp.min_str_sum;
		double min_str_neib = all_fpn ? clt_parameters.imp.min_str_neib_fpn : clt_parameters.imp.min_str_neib;
		double corr_fz_inter = clt_parameters.getGpuFatZeroInter(ref_scene.isMonochrome());
		if (mb_en && (mb_vectors!=null)) { // increase fat zero when there is motion blur
		corr_fz_inter *= 8;
		@@ -13189,12 +13190,16 @@ public class OpticalFlow {
		clt_parameters.imp.centroid_radius,// final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
		clt_parameters.imp.n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
		clt_parameters.imp.td_weight, // final double td_weight, // mix correlations accumulated in TD with
		clt_parameters.imp.td_neib_weight, // final double td_neib_weight, // mix correlations accumulated in TD (neibs)
		clt_parameters.imp.pd_weight, // final double pd_weight, // correlations (post) accumulated in PD
		clt_parameters.imp.td_nopd_only, // final boolean td_nopd_only , // only use TD accumulated data if no safe PD is available for the tile.
		clt_parameters.imp.neib_notd_only, // final boolean neib_notd_only, // use neighbors only if individual TD is too weak
		min_str, // final double min_str_nofpn, // = 0.25;
		min_str_sum, // final double min_str_sum_nofpn, // = 0.8; // 5;
		min_str_neib, // final double min_str_neib_nofpn,
		clt_parameters.imp.min_str_fpn, // final double min_str, // = 0.25;
		clt_parameters.imp.min_str_sum_fpn,// final double min_str_sum, // = 0.8; // 5;
		clt_parameters.imp.min_str_neib_fpn,//final double min_str_neib_fpn,
		clt_parameters.imp.min_neibs, // final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
		clt_parameters.imp.weight_zero_neibs, // final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
		half_disparity, // final double half_disparity, // 5.0; // Reduce weight twice for this disparity

Admin message