Skip to content

I
imagej-elphel
Commit 1ddf4e46 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

Implementing FG/BG split of far tiles with small disparity difference

parent df71e1d0
Hide whitespace changes
Inline Side-by-side
Showing with 962 additions and 153 deletions
src/main/java/com/elphel/imagej/tileprocessor/BiQuadParameters.java
View file @ 1ddf4e46
......@@ -28,6 +28,7 @@ import java.util.Properties;
import com.elphel.imagej.common.GenericJTabbedDialog;
public class BiQuadParameters {
private static final String [] FSPLIT_MODES= {"disabled","avg->fg+bg","avg->fg+bg:fg:bg"};
public boolean rig_mode_debug = true;
public boolean no_int_x0 = true; // do not offset window to integer maximum for the rig - used when averaging low textures to avoid "jumps" for very wide maximums
public boolean use_poly = true;
......@@ -249,7 +250,7 @@ public class BiQuadParameters {
// calculating GT
// now used not just for ML, but for any interscene depth maps
public double mll_min_disp_change_pre = 0.03; // stop re-measure when difference is below, no-LMA, 40 pairs
public int mll_max_refines_pre = 5;
public double mll_min_disp_change_lma = 0.003; // stop re-measure when difference is below, LMA, 120 pairs
......@@ -257,6 +258,18 @@ public class BiQuadParameters {
public int mll_max_refines_bg = 5;
public int mll_max_bg_nearest = 1; // number of selecting BG until switching to "nearest" to prevent switching to BG of BG
// FG/BG split for far tiles where dual max can only be detected with LMA as it is too small
public boolean fsplit_en = true; // enable split of far tiles into FG/BG with LMA
public int fsplit_mode = 1; // 0: disable, 1:avg->fg+bg, 2: avg->fg+fg:fg:bg
public double fsplit_str = 0.4; // minimal strength of the tile to split
public int fsplit_neibs = 3; // minimal number of neighbors to consider split
public int fsplit_neibs_side = 2; // *** minimal number of on each side (for thin FG over BG)
public double fsplit_neib_str = 0.4; // minimal strength of the neighbor to compare
public double fsplit_neib_diff = 1.5; // *** maximal disparity difference to neighbor to use
public double fsplit_disp = 3.0; // maximal tile disparity to consider split.
public double fsplit_adiff = 0.04;// minimal tile (combo) absolute disparity difference from neighbors
public double fsplit_rdiff = 0.1; // minimal tile (combo) relative disparity difference from neighbors
public double fsplit_kfg_min = 0.1; // *** minimal fraction of kfg and (1.0-kfg)
public boolean mll_generate_scene_outlines = false; // Uses 2 GB - change format, add dimensions (separate color for ref)
......@@ -703,6 +716,33 @@ public class BiQuadParameters {
"Measure BG disparity for the tiles that have dual correlation maximums");
gd.addNumericField("Number of BG passes until switching to the \"nearest\" selection", this.mll_max_bg_nearest, 0,3,"",
"only for the first BG passes select BG, later switch to \"nearest\" selection to prevent cascade FG->BG,... FG->BG switching");
gd.addMessage("FG/BG split for far tiles with LMA");
gd.addCheckbox ("Enable far FG/BG split with LMA", this.fsplit_en,
"Enable split of far tiles into FG/BG with LMA");
gd. addChoice("FG/BG split mode", FSPLIT_MODES, FSPLIT_MODES[this.fsplit_mode],
"Passes: AVG->.. (start from unresolved maximums), FG - foreground in/out, BG - background in/out");
gd.addNumericField("Minimal tile strength", this.fsplit_str, 3,6,"",
"Minimal tile strength to consider splitting into FG/BG pair.");
gd.addNumericField("Minimal neighbors", this.fsplit_neibs, 0,3,"",
"Minimal number of tile nighbors (of 8) to consider splitting.");
gd.addNumericField("Minimal neighbors on each side (for thin FG over BG)", this.fsplit_neibs_side,0,3,"",
"minimal number of on each side (for thin FG over BG).");
gd.addNumericField("Minimal neighbor strength", this.fsplit_neib_str, 3,6,"",
"Minimal tile neighbor strength to consider use for average.");
gd.addNumericField("Maximal neighbor difference from center", this.fsplit_neib_diff,3,6,"pix",
"Maximal disparity difference to neighbor from the center to use.");
gd.addNumericField("Maximal FG/BG split disparity", this.fsplit_disp, 3,6,"pix",
"Maximal tile disparity to consider split.");
gd.addNumericField("Maximal absolute disparity difference for FG/BG split", this.fsplit_adiff, 3,6,"pix",
"Maximal absolute disparity difference from average of neighbors for FG/BG split.");
gd.addNumericField("Maximal relative disparity difference for FG/BG split", this.fsplit_rdiff, 3,6,"",
"Maximal relative (to average) disparity difference from average of neighbors for FG/BG split.");
gd.addNumericField("Minimal kFG fraction", this.fsplit_kfg_min, 3,6,"",
"Minimal fraction of kfg and (1.0-kfg).");
gd.addMessage("Other");
gd.addCheckbox ("Generate scene outlines", this.mll_generate_scene_outlines,
"Generate and save scene outlines for scene series (need to change format, it is 2GB with uncompressed Tiff)");
......@@ -1007,6 +1047,19 @@ public class BiQuadParameters {
this.mll_max_refines_lma= (int) gd.getNextNumber();
this.mll_max_refines_bg= (int) gd.getNextNumber();
this.mll_max_bg_nearest= (int) gd.getNextNumber();
this.fsplit_en = gd.getNextBoolean();
this.fsplit_mode = gd.getNextChoiceIndex();
this.fsplit_str = gd.getNextNumber();
this.fsplit_neibs = (int) gd.getNextNumber();
this.fsplit_neibs_side = (int) gd.getNextNumber();
this.fsplit_neib_str = gd.getNextNumber();
this.fsplit_neib_diff = gd.getNextNumber();
this.fsplit_disp = gd.getNextNumber();
this.fsplit_adiff = gd.getNextNumber();
this.fsplit_rdiff = gd.getNextNumber();
this.fsplit_kfg_min = gd.getNextNumber();
this.mll_generate_scene_outlines= gd.getNextBoolean();
this.mll_add_combo= gd.getNextBoolean();
......@@ -1252,12 +1305,25 @@ public class BiQuadParameters {
properties.setProperty(prefix+"oc_min_disparity", this.oc_min_disparity+"");
properties.setProperty(prefix+"oc_min_strength", this.oc_min_strength+"");
properties.setProperty(prefix+"mll_min_disp_change_pre", this.mll_min_disp_change_pre+"");
properties.setProperty(prefix+"mll_max_refines_pre", this.mll_max_refines_pre+"");
properties.setProperty(prefix+"mll_min_disp_change_lma", this.mll_min_disp_change_lma+"");
properties.setProperty(prefix+"mll_max_refines_lma", this.mll_max_refines_lma+"");
properties.setProperty(prefix+"mll_max_refines_bg", this.mll_max_refines_bg+"");
properties.setProperty(prefix+"mll_max_bg_nearest", this.mll_max_bg_nearest+"");
properties.setProperty(prefix+"mll_min_disp_change_pre", this.mll_min_disp_change_pre+"");
properties.setProperty(prefix+"mll_max_refines_pre", this.mll_max_refines_pre+"");
properties.setProperty(prefix+"mll_min_disp_change_lma", this.mll_min_disp_change_lma+"");
properties.setProperty(prefix+"mll_max_refines_lma", this.mll_max_refines_lma+"");
properties.setProperty(prefix+"mll_max_refines_bg", this.mll_max_refines_bg+"");
properties.setProperty(prefix+"mll_max_bg_nearest", this.mll_max_bg_nearest+"");
properties.setProperty(prefix+"fsplit_en", this.fsplit_en+""); // boolean
properties.setProperty(prefix+"fsplit_mode", this.fsplit_mode+""); // int
properties.setProperty(prefix+"fsplit_str", this.fsplit_str+""); // double
properties.setProperty(prefix+"fsplit_neibs", this.fsplit_neibs+""); // int
properties.setProperty(prefix+"fsplit_neibs_side", this.fsplit_neibs_side+"");// int
properties.setProperty(prefix+"fsplit_neib_str", this.fsplit_neib_str+""); // double
properties.setProperty(prefix+"fsplit_neib_diff", this.fsplit_neib_diff+""); // double
properties.setProperty(prefix+"fsplit_disp", this.fsplit_disp+""); // double
properties.setProperty(prefix+"fsplit_adiff", this.fsplit_adiff+""); // double
properties.setProperty(prefix+"fsplit_rdiff", this.fsplit_rdiff+""); // double
properties.setProperty(prefix+"fsplit_kfg_min", this.fsplit_kfg_min+""); // double
properties.setProperty(prefix+"mll_generate_scene_outlines",this.mll_generate_scene_outlines+"");
properties.setProperty(prefix+"mll_add_combo", this.mll_add_combo+"");
......@@ -1506,6 +1572,19 @@ public class BiQuadParameters {
if (properties.getProperty(prefix+"mll_max_refines_lma")!=null) this.mll_max_refines_lma=Integer.parseInt(properties.getProperty(prefix+"mll_max_refines_lma"));
if (properties.getProperty(prefix+"mll_max_refines_bg")!=null) this.mll_max_refines_bg=Integer.parseInt(properties.getProperty(prefix+"mll_max_refines_bg"));
if (properties.getProperty(prefix+"mll_max_bg_nearest")!=null) this.mll_max_bg_nearest=Integer.parseInt(properties.getProperty(prefix+"mll_max_bg_nearest"));
if (properties.getProperty(prefix+"fsplit_en")!=null) this.fsplit_en=Boolean.parseBoolean(properties.getProperty(prefix+"fsplit_en"));
if (properties.getProperty(prefix+"fsplit_mode")!=null) this.fsplit_mode=Integer.parseInt(properties.getProperty(prefix+"fsplit_mode"));
if (properties.getProperty(prefix+"fsplit_str")!=null) this.fsplit_str=Double.parseDouble(properties.getProperty(prefix+"fsplit_str"));
if (properties.getProperty(prefix+"fsplit_neibs")!=null) this.fsplit_neibs=Integer.parseInt(properties.getProperty(prefix+"fsplit_neibs"));
if (properties.getProperty(prefix+"fsplit_neibs_side")!=null) this.fsplit_neibs_side=Integer.parseInt(properties.getProperty(prefix+"fsplit_neibs_side"));
if (properties.getProperty(prefix+"fsplit_neib_str")!=null) this.fsplit_neib_str=Double.parseDouble(properties.getProperty(prefix+"fsplit_neib_str"));
if (properties.getProperty(prefix+"fsplit_neib_diff")!=null) this.fsplit_neib_diff=Double.parseDouble(properties.getProperty(prefix+"fsplit_neib_diff"));
if (properties.getProperty(prefix+"fsplit_disp")!=null) this.fsplit_disp=Double.parseDouble(properties.getProperty(prefix+"fsplit_disp"));
if (properties.getProperty(prefix+"fsplit_adiff")!=null) this.fsplit_adiff=Double.parseDouble(properties.getProperty(prefix+"fsplit_adiff"));
if (properties.getProperty(prefix+"fsplit_rdiff")!=null) this.fsplit_rdiff=Double.parseDouble(properties.getProperty(prefix+"fsplit_rdiff"));
if (properties.getProperty(prefix+"fsplit_kfg_min")!=null) this.fsplit_kfg_min=Double.parseDouble(properties.getProperty(prefix+"fsplit_kfg_min"));
if (properties.getProperty(prefix+"mll_generate_scene_outlines")!=null) this.mll_generate_scene_outlines=Boolean.parseBoolean(properties.getProperty(prefix+"mll_generate_scene_outlines"));
if (properties.getProperty(prefix+"mll_add_combo")!=null) this.mll_add_combo=Boolean.parseBoolean(properties.getProperty(prefix+"mll_add_combo"));
......@@ -1754,6 +1833,19 @@ public class BiQuadParameters {
bqp.mll_max_refines_lma = this.mll_max_refines_lma;
bqp.mll_max_refines_bg = this.mll_max_refines_bg;
bqp.mll_max_bg_nearest = this.mll_max_bg_nearest;
bqp.fsplit_en = this.fsplit_en;
bqp.fsplit_mode = this.fsplit_mode;
bqp.fsplit_str = this.fsplit_str;
bqp.fsplit_neibs = this.fsplit_neibs;
bqp.fsplit_neibs_side = this.fsplit_neibs_side;
bqp.fsplit_neib_str = this.fsplit_neib_str;
bqp.fsplit_neib_diff = this.fsplit_neib_diff;
bqp.fsplit_disp = this.fsplit_disp;
bqp.fsplit_adiff = this.fsplit_adiff;
bqp.fsplit_rdiff = this.fsplit_rdiff;
bqp.fsplit_kfg_min = this.fsplit_kfg_min;
bqp.mll_generate_scene_outlines = this.mll_generate_scene_outlines;
bqp.mll_add_combo = this.mll_add_combo;
......
src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java
View file @ 1ddf4e46
......@@ -1742,6 +1742,7 @@ public class ImageDtt extends ImageDttCPU {
final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMA for the integrated correlations
final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
final double [][] rXY, // from geometryCorrection
// next both can be nulls
final double [][][][] clt_corr_out, // sparse (by the first index) [type][tilesY][tilesX][(2*transform_size-1)*(2*transform_size-1)] or null
......@@ -1957,6 +1958,7 @@ public class ImageDtt extends ImageDttCPU {
if (fcorr_td[tileY][tileX] == null) {
continue; // nothing accumulated for this tile
}
// boolean dbg_val=globalDebugLevel>100;
nTile = tileY * tilesX + tileX;
if (tp_tasks[iTile].getTask() == 0) continue; // nothing to do for this tile
boolean debugTile0 =(tileX == debug_tileX) && (tileY == debug_tileY) && (globalDebugLevel > 0); // 1);
......@@ -2090,14 +2092,33 @@ public class ImageDtt extends ImageDttCPU {
double [][] disp_str_sel = null;
double [][] disp_str_lma = null;
int [] sel_fg_bg = null;
boolean use_far_fgbg = false;
// now use older LMA (single-max) when LY is needed
if (imgdtt_params.bimax_dual_LMA && (ddnd == null)) {
double [][] maxes = correlation2d.getDoublePoly(
disparity_scale, // double disparity_scale,
((corr_dia_tile != null) ? corr_dia_tile : corr_combo_tile), // double [] combo_corrs,
imgdtt_params.mcorr_dual_fract, //double min_fraction
imgdtt_params.mcorr_dual_fract, // double min_fraction
imgdtt_params.mcorr_dual_min_max, // double min_max, // = 0.2; // Minimal absolute strength of the strongest in a dual-max to consider second one
imgdtt_params.mcorr_dual_min_min); // double min_min); // = 0.08; // Minimal absolute strength of a weakest in a dual-max to consider second one
// calculate hints for LMA when FG and BG are too close to be separated
// and separation data is provided externally
if ((far_fgbg != null) && (far_fgbg[nTile] != null)) { // only if run_lma
if (!run_lma) {
continue; // this mode is defined only for LMA
}
if (maxes.length == 1) { // normally should be just one. If 2 - OK, use old splitting (abnormal)
maxes = split_far_max(
maxes[0], // double [] max,
far_fgbg[nTile][0], // double diff,
far_fgbg[nTile][1], // double kfg,
imgdtt_params.mcorr_dual_fract); // 0.1); // double min_k);
if (maxes == null) {
continue;
}
}
use_far_fgbg = true;
}
if ((disparity_map != null) && (disparity_map[DISPARITY_VARIATIONS_INDEX] != null)) {
disparity_map[DISPARITY_VARIATIONS_INDEX ][nTile] = maxes.length;
}
......@@ -2127,15 +2148,19 @@ public class ImageDtt extends ImageDttCPU {
imgdtt_params.bimax_combine_mode, // int combine_mode, // 0 - both, 1 - strongest, 2 - nearest to zero, 3 - FG, 4 - BG
maxes); // double[][] disp_str_dual) // -preliminary center x in pixels for largest baseline
int combine_mode_corrected = imgdtt_params.bimax_combine_mode;
// Simplify - no need to backup single max - just fail, it should be already known
if ( (maxes[sel_fg_bg[0]][1] < imgdtt_params.mcorr_fb_fract * maxes[sel_fg_bg[1]][1]) ||
(maxes[sel_fg_bg[1]][1] < imgdtt_params.mcorr_bf_fract * maxes[sel_fg_bg[0]][1])) {
if (use_far_fgbg) {
continue; // requires both maximums (should not happen)
}
//maxes = new double [][] {maxes[0]}; // strongest
disp_str_sel = new double [][] {maxes[0]}; // strongest
sel_max = 0;
sel_fg_bg = new int[] {0,0};
combine_mode_corrected = Correlation2d.CAMEL_STRONGEST;
}
if (run_lma) {
if (debugTile1) {
System.out.println("clt_process_tl_correlations() maxes, tp_task["+iTile+"]target_disparity="+tp_tasks[iTile].getTargetDisparity());
......@@ -2159,7 +2184,7 @@ public class ImageDtt extends ImageDttCPU {
(debugTile0 ? 1: -2), // int debug_level,
tileX, // int tileX, // just for debug output
tileY );
if (debugTile1 && (lma_dual!= null)) { // addad (lma_dual != null)
if (debugTile1 && (lma_dual!= null)) { // added (lma_dual != null)
System.out.println("clt_process_tl_correlations() corrLMA2DualMax() done, lma_dual="+
((lma_dual== null)? "null": " not null"));
}
......@@ -2190,10 +2215,10 @@ public class ImageDtt extends ImageDttCPU {
}
}
}
if (imgdtt_params.bimax_post_LMA && (sel_max >=0)) {
if (!use_far_fgbg && imgdtt_params.bimax_post_LMA && (sel_max >=0)) { // ?? 04/09/2023 - added use_far_fgbg
disp_str_lma = new double [][] {disp_str_lma[sel_max]}; // was already selected before LMA
}
} // if (lma_dual != null) {
}
}
} // no positive maximums on y==0 - nothing in this tile
}
......@@ -2206,24 +2231,37 @@ public class ImageDtt extends ImageDttCPU {
}
}
if (disparity_map != null) {
disparity_map[DISPARITY_INDEX_CM ][nTile] = disp_str_sel[0][0]; // disparity non-LMA
disparity_map[DISPARITY_INDEX_CM + 1 ][nTile] = disp_str_sel[0][1]; // strength non-LMA;
disparity_map[DISPARITY_STRENGTH_INDEX][nTile] = disp_str_sel[0][1]; // strength non-LMA;
if ((disp_str_lma!=null) && (disp_str_lma.length>0)) {
if (disp_str_lma.length == 1) {
if (!Double.isNaN(disp_str_lma[0][0])) {
disparity_map[DISPARITY_INDEX_POLY ][nTile] = disp_str_lma[0][0]; // disparity LMA
disparity_map[DISPARITY_INDEX_POLY + 1][nTile] = disp_str_lma[0][2]; // strength LMA
// keep strength from CM strength (not LMA)
// disparity_map[DISPARITY_STRENGTH_INDEX][nTile] = disp_str_lma[0][1]; // overwrite with LMA strength
}
} else {
// In this new mode only set if there are 2 maximums.
// Set FG DS to DISPARITY_INDEX_POLY, BG DS to DISPARITY_INDEX_POLY (instead of non-LMA)
if (use_far_fgbg) {
if ((disp_str_lma!=null) && // only if both maximums are OK, otherwise keep Double.NaN
(disp_str_lma.length > 1) &&
!Double.isNaN(disp_str_lma[0][0]) &&
!Double.isNaN(disp_str_lma[1][0])) {
int indx = sel_fg_bg[0];
if (!Double.isNaN(disp_str_lma[indx][0])) {
disparity_map[DISPARITY_INDEX_POLY ][nTile] = disp_str_lma[indx][0]; // disparity LMA
disparity_map[DISPARITY_INDEX_POLY + 1][nTile] = disp_str_lma[indx][2]; // strength LMA
// keep strength from CM strength (not LMA)
// disparity_map[DISPARITY_STRENGTH_INDEX][nTile] = disp_str_lma[indx][1]; // overwrite with LMA strength
// set FG to DISPARITY_INDEX_POLY/DISPARITY_INDEX_POLY+1
disparity_map[DISPARITY_INDEX_POLY ][nTile] = disp_str_lma[indx][0]; // disparity LMA
disparity_map[DISPARITY_INDEX_POLY + 1][nTile] = disp_str_lma[indx][2]; // strength LMA
// set BG to DISPARITY_INDEX_CM/DISPARITY_INDEX_CM+1
disparity_map[DISPARITY_INDEX_CM ][nTile] = disp_str_lma[1-indx][0]; // disparity LMA
disparity_map[DISPARITY_INDEX_CM + 1][nTile] = disp_str_lma[1-indx][2]; // strength LMA
}
} else {
disparity_map[DISPARITY_INDEX_CM ][nTile] = disp_str_sel[0][0]; // disparity non-LMA
disparity_map[DISPARITY_INDEX_CM + 1 ][nTile] = disp_str_sel[0][1]; // strength non-LMA;
disparity_map[DISPARITY_STRENGTH_INDEX][nTile] = disp_str_sel[0][1]; // strength non-LMA;
if ((disp_str_lma!=null) && (disp_str_lma.length>0)) {
if (disp_str_lma.length == 1) {
if (!Double.isNaN(disp_str_lma[0][0])) {
disparity_map[DISPARITY_INDEX_POLY ][nTile] = disp_str_lma[0][0]; // disparity LMA
disparity_map[DISPARITY_INDEX_POLY + 1][nTile] = disp_str_lma[0][2]; // strength LMA
}
} else {
int indx = sel_fg_bg[0];
if (!Double.isNaN(disp_str_lma[indx][0])) {
disparity_map[DISPARITY_INDEX_POLY ][nTile] = disp_str_lma[indx][0]; // disparity LMA
disparity_map[DISPARITY_INDEX_POLY + 1][nTile] = disp_str_lma[indx][2]; // strength LMA
}
}
}
}
......@@ -2397,8 +2435,72 @@ public class ImageDtt extends ImageDttCPU {
return;
}
/**
* Helper method for clt_process_tl_correlations - split unresolved dual maximum for far (low disparity) tiles
* into a pair of FG and BG one to use as initial settings for LMA.
*
* There are 3 operation modes encoded by 0.0 for one of the diff or kfg parameters:
* a) Target disparity used for 2D phase correlations was provided for combined (unresolved) maximums.
* Calculated maximums are around the unresolved disparity, at diff distance, sum of weights equals
* to the max[1] (combined strength).
* b) Target disparity was provided for FG maximum. This is indicated by diff = 0.0. In this case
* disparity of the FG is 0.0, disparity of the BG is calculated so their center of mass is at max[0].
* c) Target disparity is provided for BG - it is calculated after FG is known, so the caller knows diff.
* This mode is indicated by kfg = 0.0. Disparity of BG is set to 0, disparity of FG - to diff, and
* weights are calculated
* @param max a pair of {disparity,strength} calculated from the correlation without resolving dual maximim.
* @param diff expected difference between FG and BG disparities provided by the caller.
* @param kfg fraction (0..1) of FG maximum weight of the full max[1] strength
* @param min_k minimal kfg and (1-kfg) if one maximum is much weaker than another
* @return a pair of {disparity, strength} pairs for FG and BG maximums as a seed for LMA, ordered starting
* with a stronger maximums (same as maxes[][] in the caller). Returns null if result does not make sense.
*/
public static double [][] split_far_max(
double [] max,
double diff,
double kfg,
double min_k){
if ((kfg > 0) && ((kfg < min_k) || ((1.0 - kfg) < min_k))) {
return null; // one of the maximums is too weak with respect to the other
}
double [][] maxes = new double [2][2];
if (diff == 0) { // target is FG
if (kfg <= 0) {
return null; //diff and kfg can not be 0.0 simultaneously (or use this case for something?)
}
if (max[0] >= 0) { // combo disparity should be negative
return null;
}
maxes[0][0] = 0.0; // disparity(FG)
maxes[1][0] = max[0] / kfg; // disparity(BG)
maxes[0][1] = max[1] * kfg; // strength(FG)
maxes[1][1] = max[1] * (1.0 - kfg); // strength(BG)
} else if (kfg == 0) { // target is BG
if ((max[0] <= 0.0) || (max[0] >= diff)) {
return null; // averaged max should have positive disparity < diff
}
maxes[0][0] = diff; // disparity(FG)
maxes[1][0] = 0.0; // disparity(BG)
maxes[0][1] = max[1] * (max[0] / diff); // strength(FG)
maxes[1][1] = max[1] * ((diff - max[0]) / diff); // strength(BG)
} else { // target is unresolved merged maximums
maxes[0][0] = max[0] + diff * (1.0 - kfg); // disparity(FG)
maxes[1][0] = max[0] - diff * kfg; // disparity(BG)
maxes[0][1] = max[1] * kfg; // strength(FG)
maxes[1][1] = max[1] * (1.0 - kfg); // strength(BG)
}
if (maxes[1][1] > maxes[0][1]) {
double [] tmp = maxes[0];
maxes[0] = maxes[1];
maxes[1]= tmp;
}
if ((maxes[0][1] / maxes[1][1]) > (1.0 - min_k)/min_k) {
return null; // too large difference between strong and weak;
}
return maxes;
}
......
src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java
View file @ 1ddf4e46
......@@ -516,8 +516,8 @@ public class IntersceneMatchParameters {
"Blur weak sigma");
gd.addNumericField("Number of blur passes", this.num_blur, 0,3,"",
"Number of blur passes");
gd.addNumericField("Disparity offset (not used)", this.disparity_corr, 5,7,"",
"for setInterTasks() - currently not used ");
gd.addNumericField("*** Disparity offset (used 04/07/2023) ***", this.disparity_corr, 5,7,"",
"for setInterTasks() - used 04/07/2023.");
gd.addNumericField("Outlier N-th from extreme", this.outliers_nth_fromextrem, 0,3,"",
"0 - use min/max, 1 - use second min/max, ... second from min/max - removes dual-tile clusters max/mins");
gd.addNumericField("Outliers tolerance absolute", this.outliers_tolerance_absolute, 5,7,"pix",
......
src/main/java/com/elphel/imagej/tileprocessor/MultisceneLY.java
View file @ 1ddf4e46
......@@ -870,7 +870,7 @@ public class MultisceneLY {
final double gpu_sigma_corr = clt_parameters.getGpuCorrSigma(last_scene.isMonochrome());
final double gpu_sigma_rb_corr = last_scene.isMonochrome()? 1.0 : clt_parameters.gpu_sigma_rb_corr;
final double gpu_sigma_log_corr = clt_parameters.getGpuCorrLoGSigma(last_scene.isMonochrome());
double disparity_corr = 0.0;
final double disparity_corr = clt_parameters.imp.disparity_corr; // 04/07/2023 // 0.0;
int debugLevel = debug_level;
final double [][] lazy_eye_data = new double [clustersY*clustersX][];
ImageDtt image_dtt= new ImageDtt(
......@@ -1029,6 +1029,7 @@ public class MultisceneLY {
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
tp_tasks_combo, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
null, // final double [][][] far_fgbg, // null, or [tilesY][tilesX]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
last_scene.getErsCorrection().getRXY(false), // final double [][] rXY, // from geometryCorrection
// next both can be nulls
null, // final double [][][][] clt_corr_out, // sparse (by the first index) [type][tilesY][tilesX][(2*transform_size-1)*(2*transform_size-1)] or null
......
src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java
View file @ 1ddf4e46
......@@ -88,6 +88,33 @@ public class OpticalFlow {
public static double [] ZERO3 = {0.0,0.0,0.0};
public static double LINE_ERR = 0.1;
public static int THREADS_MAX = 100; // maximal number of threads to launch
// 4 variants for 8 neighbors (N,NE,E...NW) to split into 2 opposite groups to compare with the center tile
public static int [][] KERN_FG = {
{1,1,3,2,2,2,3,1},
{3,2,2,2,3,1,1,1},
{1,3,2,2,2,3,1,1},
{1,1,1,3,2,2,2,3}};
public static double W = 0.25; // 0.5
public static double [][] KERN_FG_W = { // corresponding weights
{1,W,W,W,1,W,W,W},
{W,W,1,W,W,W,1,W},
{W,W,W,1,W,W,W,1},
{W,1,W,W,W,1,W,W}};
// Same for a larged kernel (5x5)
public static int [][] KERN_FG2 = {
// 0 1 2 3 4 5 6 7 8 91011121314151617181920212223
{1,1,3,2,2,2,3,1,1,1,0,0,3,0,0,2,2,2,0,0,3,0,0,1},
{3,2,2,2,3,1,1,1,3,0,0,2,2,2,0,0,3,0,0,1,1,1,0,0},
{1,3,2,2,2,3,1,1,0,0,3,0,0,2,2,2,0,0,3,0,0,1,1,1},
{1,1,1,3,2,2,2,3,0,1,1,1,0,0,3,0,0,2,2,2,0,0,3,0}};
public static double [][] KERN_FG_W2 = {
// 0 1 2 3 4 5 6 7 8 91011121314151617181920212223
{1,W,W,W,1,W,W,W,W,W,0,0,W,0,0,W,W,W,0,0,W,0,0,W},
{W,W,1,W,W,W,1,W,W,0,0,W,W,W,0,0,W,0,0,W,W,W,0,0},
{W,W,W,1,W,W,W,1,0,0,W,0,0,W,W,W,0,0,W,0,0,W,W,W},
{W,1,W,W,W,1,W,W,0,W,W,W,0,0,W,0,0,W,W,W,0,0,W,0}};
public boolean updateStatus = true;
public int numSens;
// not configured yet
......@@ -2471,6 +2498,7 @@ public class OpticalFlow {
}
public static double [][] getSceneDisparityStrength(
final CLTParameters clt_parameters,
final boolean to_ref_disparity, // false - return scene disparity, true - convert disparity back to the reference scene
final double [] disparity_ref, // invalid tiles - NaN in disparity
final double [] disparity_scene, // invalid tiles - NaN in disparity (just for masking out invalid scene tiles)
......@@ -2492,7 +2520,7 @@ public class OpticalFlow {
final int tilesY = tp.getTilesY();
final int transform_size = tp.getTileSize();
final int tiles = tilesX*tilesY;
final double scene_disparity_cor = 0.0;
final double scene_disparity_cor = clt_parameters.imp.disparity_corr; // 04/07/2023 // 0.0;
final boolean [] valid_tiles = new boolean[tiles];
scene_QuadClt.getErsCorrection().setErsDt(
scene_ers_xyz_dt, // double [] ers_xyz_dt,
......@@ -7637,6 +7665,7 @@ public class OpticalFlow {
indx_ref, // final int indx_ref,
combo_dsn_change[0], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
null, // final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
margin, // final int margin,
nrefine, // final int nrefine, // just for debug title
clt_parameters.inp.show_final_2d, // final boolean show_2d_corr,
......@@ -7907,7 +7936,7 @@ public class OpticalFlow {
return imp_stack;
}
public double [][] intersceneExport(
public static double [][] intersceneExport(
CLTParameters clt_parameters,
ErsCorrection ers_reference,
QuadCLT [] scenes,
......@@ -7915,6 +7944,25 @@ public class OpticalFlow {
int debug_level
)
{
final boolean fsplit_en = clt_parameters.rig.fsplit_en; // true; // enable split of far tiles into FG/BG with LMA
final int fsplit_mode = clt_parameters.rig.fsplit_mode; // 1; // 0: disable, 1:avg->fg + fg->fg +fg->bg, 2: avg->fg+fg->bg, 3: avg->fg, avg->bg
final double fsplit_str = clt_parameters.rig.fsplit_str; // 0.4; // minimal strength of the tile to split
final int fsplit_neibs = clt_parameters.rig.fsplit_neibs; // 3; // minimal n umber of neighbors to consider split
final int fsplit_neibs_side =clt_parameters.rig.fsplit_neibs_side;// 2; // minimal number of on each side (for thin FG over BG)
final double fsplit_neib_str = clt_parameters.rig.fsplit_neib_str; // 0.4; // minimal strength of the neighbor to compare
final double fsplit_neib_diff = clt_parameters.rig.fsplit_neib_diff; // 1.5; // maximal disparity difference to neighbor to use
final double fsplit_disp = clt_parameters.rig.fsplit_disp; // 3.0; // maximal tile disparity to consider split.
final double fsplit_adiff = clt_parameters.rig.fsplit_adiff; // 0.08;// minimal tile (combo) absolute disparity difference from neighbors
final double fsplit_rdiff = clt_parameters.rig.fsplit_rdiff; // 0.1; // minimal tile (combo) relative disparity difference from neighbors
final double fsplit_kfg_min = clt_parameters.rig.fsplit_kfg_min; // 0.1; // minimal fraction of kfg and (1.0-kfg)
int fsplit_refines = 0;
if (fsplit_en) switch (fsplit_mode) {
case 1: fsplit_refines = 1; break;
case 2: fsplit_refines = 3; break;
}
final double scale_far_split_strength = 1.5;
// empirical correction for both lma and non-lma step
double corr_nonlma = 1.0; // 1.23;
double corr_lma = 1.0; // 1.23;
......@@ -7935,11 +7983,11 @@ public class OpticalFlow {
}
double min_disp_change = clt_parameters.rig.mll_min_disp_change_pre; // 0.001; // stop re-measure when difference is below
final int max_refines =
final int max_refines_presplit =
clt_parameters.rig.mll_max_refines_bg +
clt_parameters.rig.mll_max_refines_lma +
clt_parameters.rig.mll_max_refines_pre;
final int max_refines = max_refines_presplit + fsplit_refines;
final int [] iter_indices = {
COMBO_DSN_INDX_DISP,
COMBO_DSN_INDX_STRENGTH,
......@@ -7952,10 +8000,6 @@ public class OpticalFlow {
double [][] combo_dsn = null;
// USE reference scene if no individual DSI are available double [][] dsrbg = scene.getDSRBG(); Probably null for non-reference
// make sure to use already integrated if available
/// double [][] combo_dsn0 = scenes[indx_ref].readDoubleArrayFromModelDirectory(
/// "-INTER-INTRA-LMA", // String suffix,
/// 0, // int num_slices, // (0 - all)
/// null); // int [] wh);
double [][] combo_dsn0 =scenes[indx_ref].readComboDSI(true); // scenes[indx_ref].dsi and blue sky are NOT updated
......@@ -8020,6 +8064,7 @@ public class OpticalFlow {
for (int i = 0; i < combo_dsn.length; i++) { // 4 elements: "disp", "strength","disp_lma","num_valid"
if (combo_dsn[i] != null) combo_dsn_change[i] = combo_dsn[i]; // all but change
}
if (debug_level > 0) {
ShowDoubleFloatArrays.showArrays(
combo_dsn_change,
......@@ -8073,6 +8118,9 @@ public class OpticalFlow {
if (clt_parameters.rig.mll_max_refines_bg > 0) {
clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_FG; // initially refine FG
clt_parameters.img_dtt.bimax_dual_only = false; // not just dual only
if (debug_level > -3) {
System.out.println("intersceneExport(): set CAMEL_FG");
}
}
// FIXME: uncomment next
......@@ -8103,6 +8151,12 @@ public class OpticalFlow {
// disparity in each scene image set that correspond to the reference scene uniform grid tile.
// First use reduced number of pairs and no LMA, then continue with all pairs and LMA
boolean bg_refine= false;
final double [][] far_fgbg = new double [selection.length][];
final double [][] pre_split_ds = new double [2][];
boolean [] sel_split = new boolean [selection.length];
double [][] far_fg_ds = new double[2][];
double [][] far_bg_ds = new double[2][];
for (int nrefine = 0; nrefine < max_refines; nrefine++) {
if (nrefine == clt_parameters.rig.mll_max_refines_pre) {
min_disp_change = clt_parameters.rig.mll_min_disp_change_lma;
......@@ -8117,18 +8171,73 @@ public class OpticalFlow {
System.out.println ("a. nrefine pass = "+nrefine+", remaining "+num_tomeas+" tiles to re-measure");
}
} else if (nrefine == (clt_parameters.rig.mll_max_refines_pre + clt_parameters.rig.mll_max_refines_lma)) {
// Processing BG - single pass
clt_parameters.img_dtt.bimax_dual_only = true; // Camel-max tiles only
clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_BG;
bg_refine= true;
selection = selection_orig.clone(); // re-select all original tiles
if (debug_level > -3) {
System.out.println("intersceneExport(): set CAMEL_BG, bimax_dual_only=true @"+nrefine);
}
} else if (nrefine == (clt_parameters.rig.mll_max_refines_pre + clt_parameters.rig.mll_max_refines_lma + 1)) {
clt_parameters.img_dtt.bimax_dual_only = false; // not just dual only
if (debug_level > -3) {
System.out.println("intersceneExport(): set bimax_dual_only=false @"+nrefine);
}
}
if (nrefine == (clt_parameters.rig.mll_max_refines_pre + clt_parameters.rig.mll_max_refines_lma + clt_parameters.rig.mll_max_bg_nearest)) {
clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_NEAREST;
if (debug_level > -3) {
System.out.println("intersceneExport(): set CAMEL_NEAREST @"+nrefine);
}
}
int split_pass = nrefine - max_refines_presplit;
double [][] avg_ds = new double[][] {
combo_dsn_final[COMBO_DSN_INDX_DISP],
combo_dsn_final[COMBO_DSN_INDX_STRENGTH]};
if (split_pass >= 0) {
int split_src = 0; // avg
if (split_pass == 0) {
split_src= 0 ; // AVG
// 04/11/2023: always outputs both FG and BG
clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_FG;
sel_split = calcFarFgBg(
split_src, // final int split_src, // target disparity is for 0 - avg, 1 - fg, 2 - bg
fsplit_str, // final double fsplit_str, // minimal strength of the tile to split
fsplit_neibs, // final int fsplit_neibs, // minimal number of neighbors to consider split
fsplit_neibs_side,// final int fsplit_neibs_side,// minimal number of neighbors from each side to consider thin FG
fsplit_neib_str, // final double fsplit_neib_str, // minimal strength of the neighbor to use
fsplit_neib_diff,// final double fsplit_neib_diff,// maximal disparity difference to neighbor to use
fsplit_disp, // final double fsplit_disp, // maximal tile disparity to consider split, also for neibs.
fsplit_adiff, // final double fsplit_adiff, // minimal tile (combo) absolute disparity difference from neighbors
fsplit_rdiff, // final double fsplit_rdiff, // minimal tile (combo) relative disparity difference from neighbors
fsplit_kfg_min, // final double fsplit_kfg_min, // minimal fraction of kfg and (1.0-kfg)
far_fgbg, // final double [][] far_fgbg, // should be initialized to [nTiles][?]
avg_ds, // final double [][] avg_ds, // always defined, used for selection
null, // final double [][] fg_ds,
null, // final double [][] bg_ds,
null, // final boolean [] selection, // may be null, does not apply to neighbors
tilesX, // final int tilesX,
ref_scene.getImageName()+"-FGBG"); // final String dbg_title);
selection = sel_split; // for correlateInterscene()
} else if (split_pass == 1) {
// 04/11/2023: always outputs both FG and BG
split_src = 1; clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_FG; break;
/*
switch (fsplit_mode) {
case 1: split_src = 1; clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_FG; break;
case 2: split_src = 1; clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_BG; break;
case 3: split_src = 0; clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_BG; break;
}
*/
} else if (split_pass == 2) { // only for fsplit_mode == 1
// 04/11/2023: always outputs both FG and BG
split_src = 2; clt_parameters.img_dtt.bimax_combine_mode = Correlation2d.CAMEL_BG;
}
// boolean [] sel_fgbg =
}
int mcorr_sel = Correlation2d.corrSelEncode(clt_parameters.img_dtt,num_sensors);
double [][] disparity_map =
correlateInterscene( // should skip scenes w/o orientation 06/29/2022
......@@ -8137,14 +8246,15 @@ public class OpticalFlow {
indx_ref, // final int indx_ref,
target_disparity, // combo_dsn_change[combo_dsn_indx_disp], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid)
selection, // final boolean [] selection, // may be null, if not null do not process unselected tiles
far_fgbg, // final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
margin, // final int margin,
nrefine, // final int nrefine, // just for debug title
false, // ( nrefine == (max_refines - 1)) && clt_parameters.inp.show_final_2d, // final boolean show_2d_corr,
mcorr_sel, // final int mcorr_sel, // =
mcorr_sel, // final int mcorr_sel, // =
null, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
false, // final boolean no_map, // do not generate disparity_map (time-consuming LMA)
debug_level-8); // final int debug_level)
if (debug_level > 0) { //-3) {
ShowDoubleFloatArrays.showArrays(
disparity_map,
......@@ -8159,104 +8269,261 @@ public class OpticalFlow {
double [] map_disparity = disparity_map[ImageDtt.DISPARITY_INDEX_CM]; // 2
double [] map_strength = disparity_map[ImageDtt.DISPARITY_STRENGTH_INDEX]; // 10
double [] map_disparity_lma = disparity_map[ImageDtt.DISPARITY_INDEX_POLY]; // 8
int num_tomeas = 0; // number of tiles to measure
Arrays.fill(selection, false);
for (int nTile =0; nTile < combo_dsn_change[0].length; nTile++) {
if (defined_tiles[nTile]) { // originally defined, maybe not measured last time
if (!Double.isNaN(map_disparity[nTile])) { // re-measured
boolean is_lma = (map_disparity_lma != null) && !Double.isNaN(map_disparity_lma[nTile]);
if (is_lma) {
combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile] = map_disparity_lma[nTile] * corr_nonlma;
} else if (!Double.isNaN(map_disparity[nTile])) {
combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile] = map_disparity[nTile] / clt_parameters.ofp.magic_scale * corr_lma;
}
if (!Double.isNaN(combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile])) {
// will keep previous new_corr_scale value when no previous data or switching non-LMA/LMA. Should be initialized to 1.0
if (!Double.isNaN(disp_err[nTile]) && (is_lma == was_lma[nTile])) {
// current target_disparity: combo_dsn_change[combo_dsn_indx_disp][nTile]
double this_target_disparity = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
double previous_target_disparity = this_target_disparity - corr_scale[nTile] * disp_err[nTile];
double prev_disp_err = disp_err[nTile];
double this_disp_err = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
double new_corr_scale = (previous_target_disparity - this_target_disparity) / (this_disp_err - prev_disp_err);
if ((new_corr_scale <= max_scale) && (new_corr_scale >= min_scale)) {
corr_scale[nTile] = new_corr_scale;
}
}
// only for far split.
double [] map_far_fg_disparity = disparity_map[ImageDtt.DISPARITY_INDEX_POLY];
double [] map_far_fg_strength = disparity_map[ImageDtt.DISPARITY_INDEX_POLY+1];
double [] map_far_bg_disparity = disparity_map[ImageDtt.DISPARITY_INDEX_CM];
double [] map_far_bg_strength = disparity_map[ImageDtt.DISPARITY_INDEX_CM+1];
if (split_pass >= 0) {
int nTiles = selection.length;
boolean [] far_split = new boolean [nTiles];
// double [][] far_fg_ds = new double[2][];
// double [][] far_bg_ds = new double[2][];
double [][] far_fg_ds_merge = new double[2][];
double [][] far_bg_ds_merge = new double[2][];
far_fg_ds[0] = new double[nTiles];
far_bg_ds[0] = new double[nTiles];
Arrays.fill(far_fg_ds[0], Double.NaN);
Arrays.fill(far_bg_ds[0], Double.NaN);
far_fg_ds_merge[0] = avg_ds[0].clone();
far_bg_ds_merge[0] = avg_ds[0].clone();
far_fg_ds_merge[1] = avg_ds[1].clone();
far_bg_ds_merge[1] = avg_ds[1].clone();
// single-threaded
for (int tile = 0; tile < nTiles; tile++) {
if (!Double.isNaN(map_far_fg_disparity[tile]) && !Double.isNaN(map_far_bg_disparity[tile])) {
far_split[tile] = true;
far_fg_ds[0][tile] = avg_ds[0][tile] + map_far_fg_disparity[tile];
far_bg_ds[0][tile] = avg_ds[0][tile] + map_far_bg_disparity[tile];
far_fg_ds_merge[0][tile] = far_fg_ds[0][tile];
far_bg_ds_merge[0][tile] = far_bg_ds[0][tile];
far_fg_ds_merge[1][tile] = map_far_fg_strength[tile];
far_bg_ds_merge[1][tile] = map_far_bg_strength[tile];
}
}
far_fg_ds[1] = map_far_fg_strength;
far_bg_ds[1] = map_far_bg_strength;
if (debug_level > 0) { //-3) {
double [][] predicted_fg_ds = new double[2][nTiles];
double [][] predicted_bg_ds = new double[2][nTiles];
double [][] predicted_fg_ds_merge = new double[2][];
double [][] predicted_bg_ds_merge = new double[2][];
// predicted_fg_ds[0] = new double[nTiles];
// predicted_bg_ds[0] = new double[nTiles];
Arrays.fill(predicted_fg_ds[0], Double.NaN);
Arrays.fill(predicted_bg_ds[0], Double.NaN);
predicted_fg_ds_merge[0]= avg_ds[0].clone();
predicted_bg_ds_merge[0]= avg_ds[0].clone();
predicted_fg_ds_merge[1]= avg_ds[1].clone();
predicted_bg_ds_merge[1]= avg_ds[1].clone();
double [][] fg_predicted_err = new double[2][nTiles];
double [][] bg_predicted_err = new double[2][nTiles];
Arrays.fill(fg_predicted_err[0], Double.NaN);
Arrays.fill(bg_predicted_err[0], Double.NaN);
Arrays.fill(fg_predicted_err[1], Double.NaN);
Arrays.fill(bg_predicted_err[1], Double.NaN);
for (int tile = 0; tile < nTiles; tile++) {
if (far_fgbg[tile] != null) {
double diff = far_fgbg[tile][0];
double kfg = far_fgbg[tile][1];
/*
if (diff == 0) { // target is FG
dbg_img[ 3][tile] = 0.0 + fg_ds[0][tile]; // disparity(FG)
dbg_img[ 4][tile] = (avg_ds[0][tile] - fg_ds[0][tile]) / kfg + fg_ds[0][tile]; // disparity(BG)
dbg_img[10][tile] = avg_ds[1][tile] * kfg; // strength(FG)
dbg_img[11][tile] = avg_ds[1][tile] * (1.0 - kfg); // strength(BG)
} else if (kfg == 0) { // target is BG
dbg_img[ 3][tile] = diff + bg_ds[0][tile]; // disparity(FG)
dbg_img[ 4][tile] = 0.0 + bg_ds[0][tile]; // disparity(BG)
dbg_img[10][tile] = avg_ds[1][tile] * ((avg_ds[0][tile] - bg_ds[0][tile]) / diff); // strength(FG)
dbg_img[11][tile] = avg_ds[1][tile] * ((diff - (avg_ds[0][tile] - bg_ds[0][tile])) / diff); // strength(BG)
} else { // target is unresolved merged maximums
*/
predicted_fg_ds[0][tile] = 0 + diff * (1.0 - kfg) + avg_ds[0][tile]; // disparity(FG)
predicted_bg_ds[0][tile] = 0 - diff * kfg + avg_ds[0][tile]; // disparity(BG)
predicted_fg_ds[1][tile] = avg_ds[1][tile] * kfg; // strength(FG)
predicted_bg_ds[1][tile] = avg_ds[1][tile] * (1.0 - kfg); // strength(BG)
combo_dsn_change[COMBO_DSN_INDX_DISP][nTile] += combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile] * corr_scale[nTile];
was_lma[nTile] = is_lma;
if (!Double.isNaN(far_fg_ds[0][tile]) && !Double.isNaN(far_fg_ds[0][tile])) {
fg_predicted_err[0][tile] = predicted_fg_ds[0][tile] - far_fg_ds[0][tile];
bg_predicted_err[0][tile] = predicted_bg_ds[0][tile] - far_bg_ds[0][tile];
fg_predicted_err[1][tile] = predicted_fg_ds[1][tile] - far_fg_ds[1][tile];
bg_predicted_err[1][tile] = predicted_bg_ds[1][tile] - far_bg_ds[1][tile];
}
combo_dsn_change[COMBO_DSN_INDX_STRENGTH][nTile] = map_strength[nTile]; // combine CM/LMA
if (bg_refine) {
combo_dsn_final[COMBO_DSN_INDX_DISP_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
combo_dsn_final[COMBO_DSN_INDX_STRENGTH_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_STRENGTH][nTile];
combo_dsn_final[COMBO_DSN_INDX_LMA_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
if (map_disparity_lma != null) {
combo_dsn_final[COMBO_DSN_INDX_LMA_BG][nTile] = Double.isNaN(map_disparity_lma[nTile])? Double.NaN : combo_dsn_final[COMBO_DSN_INDX_DISP_BG][nTile];
predicted_fg_ds_merge[0][tile] = 0 + diff * (1.0 - kfg) + avg_ds[0][tile]; // disparity(FG)
predicted_bg_ds_merge[0][tile] = 0 - diff * kfg + avg_ds[0][tile]; // disparity(BG)
predicted_fg_ds_merge[1][tile] = avg_ds[1][tile] * kfg; // strength(FG)
predicted_bg_ds_merge[1][tile] = avg_ds[1][tile] * (1.0 - kfg); // strength(BG)
/*
}
*/
}
}
String [] dbg_far_spit_titles={
"avg_disp", // 0
"fg_disp", // 1
"bg_disp", // 2
"fg_disp_only", // 3
"bg_disp_only", // 4
"fg_disp_predicted_only",// 5
"bg_disp_predicted_only",// 6
"fg_disp_predicted_err", // 7
"bg_disp_predicted_err", // 8
"avg_str", // 9
"fg_str", // 10
"bg_str", // 11
"fg_str_only", // 12
"bg_str_only", // 13
"fg_str_predicted_only", // 14
"bg_str_predicted_only", // 15
"fg_str_predicted_err", // 16
"bg_str_predicted_err", // 17
"sel"}; // 16
double [][] dbg_img = new double[dbg_far_spit_titles.length][];
dbg_img[ 0] = avg_ds[0];
dbg_img[ 1] = far_fg_ds_merge[0];
dbg_img[ 2] = far_bg_ds_merge[0];
dbg_img[ 3] = far_fg_ds[0];
dbg_img[ 4] = far_bg_ds[0];
dbg_img[ 5] = predicted_fg_ds[0];
dbg_img[ 6] = predicted_bg_ds[0];
dbg_img[ 7] = fg_predicted_err[0];
dbg_img[ 8] = bg_predicted_err[0];
dbg_img[ 9] = avg_ds[1];
dbg_img[10] = far_fg_ds_merge[1];
dbg_img[11] = far_bg_ds_merge[1];
dbg_img[12] = far_fg_ds[1];
dbg_img[13] = far_bg_ds[1];
dbg_img[14] = predicted_fg_ds[1];
dbg_img[15] = predicted_bg_ds[1];
dbg_img[16] = fg_predicted_err[1];
dbg_img[17] = bg_predicted_err[1];
dbg_img[18] = new double [nTiles];
for (int tile = 0; tile < nTiles; tile++) {
// update index below !
dbg_img[18][tile] = (selection[tile] ? 0.5:0.0) + (far_split[tile] ? 0.6:0.0);
}
ShowDoubleFloatArrays.showArrays(
dbg_img,
tilesX,
tilesY,
true,
"far-split"+nrefine,
dbg_far_spit_titles);
}
} else { //if (split_pass >= 0) { // pre-far_fgbg disparity processing
int num_tomeas = 0; // number of tiles to measure
Arrays.fill(selection, false);
for (int nTile =0; nTile < combo_dsn_change[0].length; nTile++) {
if (defined_tiles[nTile]) { // originally defined, maybe not measured last time
if (!Double.isNaN(map_disparity[nTile])) { // re-measured
boolean is_lma = (map_disparity_lma != null) && !Double.isNaN(map_disparity_lma[nTile]);
if (is_lma) {
combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile] = map_disparity_lma[nTile] * corr_nonlma;
} else if (!Double.isNaN(map_disparity[nTile])) {
combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile] = map_disparity[nTile] / clt_parameters.ofp.magic_scale * corr_lma;
}
if (!Double.isNaN(combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile])) {
// will keep previous new_corr_scale value when no previous data or switching non-LMA/LMA. Should be initialized to 1.0
if (!Double.isNaN(disp_err[nTile]) && (is_lma == was_lma[nTile])) {
// current target_disparity: combo_dsn_change[combo_dsn_indx_disp][nTile]
double this_target_disparity = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
double previous_target_disparity = this_target_disparity - corr_scale[nTile] * disp_err[nTile];
double prev_disp_err = disp_err[nTile];
double this_disp_err = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
double new_corr_scale = (previous_target_disparity - this_target_disparity) / (this_disp_err - prev_disp_err);
if ((new_corr_scale <= max_scale) && (new_corr_scale >= min_scale)) {
corr_scale[nTile] = new_corr_scale;
}
}
combo_dsn_final[COMBO_DSN_INDX_CHANGE_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
} else {
combo_dsn_final[COMBO_DSN_INDX_DISP][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
combo_dsn_final[COMBO_DSN_INDX_STRENGTH][nTile] = combo_dsn_change[COMBO_DSN_INDX_STRENGTH][nTile];
combo_dsn_final[COMBO_DSN_INDX_LMA][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
if (map_disparity_lma != null) {
combo_dsn_final[COMBO_DSN_INDX_LMA][nTile] = Double.isNaN(map_disparity_lma[nTile])? Double.NaN : combo_dsn_final[COMBO_DSN_INDX_DISP][nTile];
combo_dsn_change[COMBO_DSN_INDX_DISP][nTile] += combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile] * corr_scale[nTile];
was_lma[nTile] = is_lma;
combo_dsn_change[COMBO_DSN_INDX_STRENGTH][nTile] = map_strength[nTile]; // combine CM/LMA
if (bg_refine) {
combo_dsn_final[COMBO_DSN_INDX_DISP_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
combo_dsn_final[COMBO_DSN_INDX_STRENGTH_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_STRENGTH][nTile];
combo_dsn_final[COMBO_DSN_INDX_LMA_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
if (map_disparity_lma != null) {
combo_dsn_final[COMBO_DSN_INDX_LMA_BG][nTile] = Double.isNaN(map_disparity_lma[nTile])? Double.NaN : combo_dsn_final[COMBO_DSN_INDX_DISP_BG][nTile];
}
combo_dsn_final[COMBO_DSN_INDX_CHANGE_BG][nTile] = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
} else {
combo_dsn_final[COMBO_DSN_INDX_DISP][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
combo_dsn_final[COMBO_DSN_INDX_STRENGTH][nTile] = combo_dsn_change[COMBO_DSN_INDX_STRENGTH][nTile];
combo_dsn_final[COMBO_DSN_INDX_LMA][nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile];
if (map_disparity_lma != null) {
combo_dsn_final[COMBO_DSN_INDX_LMA][nTile] = Double.isNaN(map_disparity_lma[nTile])? Double.NaN : combo_dsn_final[COMBO_DSN_INDX_DISP][nTile];
}
combo_dsn_final[COMBO_DSN_INDX_VALID][nTile] = combo_dsn[COMBO_DSN_INDX_VALID][nTile]; // not much sense
combo_dsn_final[COMBO_DSN_INDX_CHANGE][nTile] = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
}
combo_dsn_final[COMBO_DSN_INDX_VALID][nTile] = combo_dsn[COMBO_DSN_INDX_VALID][nTile]; // not much sense
combo_dsn_final[COMBO_DSN_INDX_CHANGE][nTile] = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
}
}
disp_err[nTile] = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
if (Math.abs(combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile]) >= min_disp_change) {
target_disparity[nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile] ;
selection[nTile] = true;
num_tomeas ++;
} else {
disp_err[nTile] = combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile];
if (Math.abs(combo_dsn_change[COMBO_DSN_INDX_CHANGE][nTile]) >= min_disp_change) {
target_disparity[nTile] = combo_dsn_change[COMBO_DSN_INDX_DISP][nTile] ;
selection[nTile] = true;
num_tomeas ++;
} else {
num_tomeas+=0;
}
} else { // originally defined, but not re-measured
num_tomeas+=0;
}
} else { // originally defined, but not re-measured
num_tomeas+=0;
}
}
}
if (dbg_corr_scale != null) {
dbg_corr_scale[nrefine] = corr_scale.clone();
}
// Copy disparity to disparity_lma and just mask out tiles with no LMA data (keep all if LMA did not run at all)
System.arraycopy(combo_dsn_change[COMBO_DSN_INDX_DISP], 0, combo_dsn_change[COMBO_DSN_INDX_LMA], 0, combo_dsn_change[COMBO_DSN_INDX_DISP].length); // lma
if (map_disparity_lma != null) {
for (int i = 0; i < map_disparity_lma.length; i++) {
if (Double.isNaN(map_disparity_lma[i])) {
combo_dsn_change[COMBO_DSN_INDX_LMA][i] = Double.NaN;
if (dbg_corr_scale != null) {
dbg_corr_scale[nrefine] = corr_scale.clone();
}
// Copy disparity to disparity_lma and just mask out tiles with no LMA data (keep all if LMA did not run at all)
System.arraycopy(combo_dsn_change[COMBO_DSN_INDX_DISP], 0, combo_dsn_change[COMBO_DSN_INDX_LMA], 0, combo_dsn_change[COMBO_DSN_INDX_DISP].length); // lma
if (map_disparity_lma != null) {
for (int i = 0; i < map_disparity_lma.length; i++) {
if (Double.isNaN(map_disparity_lma[i])) {
combo_dsn_change[COMBO_DSN_INDX_LMA][i] = Double.NaN;
}
}
}
}
for (int i = 0; i < iter_indices.length; i++) {
refine_results[last_initial_slices + (i * max_refines) + nrefine] = combo_dsn_change[iter_indices[i]].clone();
}
// clt_parameters.inp.show_final_2d, // final boolean show_2d_corr,
if (debug_level >0) {
ShowDoubleFloatArrays.showArrays(
combo_dsn_change,
tilesX,
tilesY,
true,
"combo_dsn-"+nrefine+"-"+ref_scene.getImageName(),
combo_dsn_titles); // dsrbg_titles);
}
if (debug_level > -2) {
System.out.println ("b. nrefine pass = "+nrefine+", remaining "+num_tomeas+" tiles to re-measure");
}
if (num_tomeas == 0) {
break;
}
for (int i = 0; i < iter_indices.length; i++) {
refine_results[last_initial_slices + (i * max_refines) + nrefine] = combo_dsn_change[iter_indices[i]].clone();
}
// clt_parameters.inp.show_final_2d, // final boolean show_2d_corr,
if (debug_level >0) {
ShowDoubleFloatArrays.showArrays(
combo_dsn_change,
tilesX,
tilesY,
true,
"combo_dsn-"+nrefine+"-"+ref_scene.getImageName(),
combo_dsn_titles); // dsrbg_titles);
}
if (debug_level > -2) {
System.out.println ("b. nrefine pass = "+nrefine+", clt_parameters.correlate_lma="+clt_parameters.correlate_lma+", remaining "+num_tomeas+" tiles to re-measure");
}
if (num_tomeas == 0) {
break;
}
} // else { //if (split_pass >= 0) {
} //for (int nrefine = 0; nrefine < max_refines; nrefine++) {
// Add duplicate of FG disparity and FG+BG disparity (FG where no BG) for visual comparison
if (clt_parameters.rig.mll_max_refines_bg > 0) {
for (int nTile =0; nTile < combo_dsn_change[0].length; nTile++) {
......@@ -8268,6 +8535,28 @@ public class OpticalFlow {
}
}
}
// add far split if available
boolean mod_cumul_disp = true;
boolean mod_cumul_bg_disp = true;
if (far_fg_ds[0] != null) {
for (int nTile =0; nTile < combo_dsn_change[0].length; nTile++) {
if (!Double.isNaN(far_fg_ds[0][nTile])) {
combo_dsn_final[COMBO_DSN_INDX_DISP_FG][nTile] = far_fg_ds[0][nTile];
if (mod_cumul_disp) {
combo_dsn_final[COMBO_DSN_INDX_DISP][nTile] = far_fg_ds[0][nTile];
}
combo_dsn_final[COMBO_DSN_INDX_STRENGTH][nTile] = far_fg_ds[1][nTile] * scale_far_split_strength;
}
if (!Double.isNaN(far_bg_ds[0][nTile])) {
combo_dsn_final[COMBO_DSN_INDX_DISP_BG_ALL][nTile] = far_bg_ds[0][nTile];
if (mod_cumul_bg_disp) {
combo_dsn_final[COMBO_DSN_INDX_DISP_BG][nTile] = far_bg_ds[0][nTile];
}
combo_dsn_final[COMBO_DSN_INDX_STRENGTH_BG][nTile] = far_bg_ds[1][nTile] * scale_far_split_strength;
}
}
}
// add blue sky slice
/*
boolean [] blue_sky = ref_scene.getBlueSky();
......@@ -8386,6 +8675,327 @@ public class OpticalFlow {
return combo_dsn_final;
}
public static boolean [] calcFarFgBg(
final int split_src, // target disparity is for 0 - avg, 1 - fg, 2 - bg
final double fsplit_str, // minimal strength of the tile to split
final int fsplit_neibs, // minimal number of neighbors to consider split
final int fsplit_neibs_side,// minimal number of neighbors from each side to consider thin FG
final double fsplit_neib_str, // minimal strength of the neighbor to use
final double fsplit_neib_diff,// maximal disparity difference to neighbor to use
final double fsplit_disp, // maximal tile disparity to consider split, also for neibs.
final double fsplit_adiff, // minimal tile (combo) absolute disparity difference from neighbors
final double fsplit_rdiff, // minimal tile (combo) relative disparity difference from neighbors
final double fsplit_kfg_min, // minimal fraction of kfg and (1.0-kfg)
final double [][] far_fgbg, // should be initialized to [nTiles][?]
final double [][] avg_ds, // always defined, used for selection
final double [][] fg_ds,
final double [][] bg_ds,
final boolean [] selection, // may be null, does not aspply to neighbors
final int tilesX,
final String dbg_title){
// final int dbg_tileX = -65, dbg_tileY=26, dbg_dX=4, dbg_dY=4; // 67,25
final int dbg_tileX = 28, dbg_tileY=22, dbg_dX=4, dbg_dY=4; // 67,25
final int nTiles = far_fgbg.length;
final int tilesY = nTiles/tilesX;
final TileNeibs tn = new TileNeibs(tilesX,tilesY);
Arrays.fill(far_fgbg, null);
final boolean [] sel_out = new boolean [nTiles];
final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
final AtomicInteger ai = new AtomicInteger(0);
final boolean [] dbg_thin_fg = (dbg_title != null) ? new boolean[nTiles]: null;
if (split_src == 0) { // other modes should have non-null selection[]
//TODO: require no-BG
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
double [][] neib_ds = new double[24][];
for (int tile = ai.getAndIncrement(); tile < nTiles; tile = ai.getAndIncrement()) {
if (dbg_title != null) {
int tileX = tile % tilesX;
int tileY = tile / tilesX;
if ( (tileX >= dbg_tileX) &&
(tileY >= dbg_tileY) &&
(tileX < (dbg_tileX + dbg_dX)) &&
(tileY < (dbg_tileY + dbg_dY))){
System.out.println("calcFarFgBg()-0: tile = "+tile+
" (X="+tileX+", Y="+tileY+")");
}
}
// TODO: maybe compare to avg_ds[0][tile] instead of fg_ds[0][tile]?
if ((bg_ds != null) && !Double.isNaN(bg_ds[0][tile]) && (bg_ds[0][tile] != fg_ds[0][tile])){
continue; // should not have BG
}
double center_disp = avg_ds[0][tile];
double center_str = avg_ds[1][tile];
if ( (center_disp <= fsplit_disp) && // processes NaN
(center_str >= fsplit_str) &&
((selection == null) || selection[tile])) {
// get neibs
Arrays.fill(neib_ds, null);
int num_neibs = 0;
for (int dir = 0; dir < 8; dir++) { // start withg 3x3
int tile1 = tn.getNeibIndex2(tile, dir);
if ( (tile1 >= 0) &&
(avg_ds[0][tile1] <= fsplit_disp) &&
(avg_ds[1][tile1] >= fsplit_neib_str) &&
(Math.abs(avg_ds[0][tile1] - center_disp) <= fsplit_neib_diff)) {
neib_ds[dir] = new double[] {avg_ds[0][tile1],avg_ds[1][tile1]};
num_neibs++;
}
}
if (num_neibs >= fsplit_neibs) {
for (int dir = 8; dir < 24; dir++) { // copy outside tiles of 4x4 kernels
int tile1 = tn.getNeibIndex2(tile, dir);
if ( (tile1 >= 0) &&
(avg_ds[0][tile1] <= fsplit_disp) &&
(avg_ds[1][tile1] >= fsplit_neib_str) &&
(Math.abs(avg_ds[0][tile1] - center_disp) <= fsplit_neib_diff)) {
neib_ds[dir] = new double[] {avg_ds[0][tile1],avg_ds[1][tile1]};
}
}
// detect thin FG over BG
int [][] num_side = new int [KERN_FG.length][3];
double [][] sw_side=new double[KERN_FG.length][3], swd_side=new double[KERN_FG.length][3], sww_side=new double[KERN_FG.length][3];
double [][] sw_side2=new double[KERN_FG.length][3], swd_side2=new double[KERN_FG.length][3];
for (int dir = 0; dir < 8; dir++) if (neib_ds[dir]!=null){
for (int i = 0; i < KERN_FG.length; i++) if (KERN_FG[i][dir] > 0){
int indx = KERN_FG[i][dir] - 1;
num_side[i][indx]++;
sww_side[i][indx] += KERN_FG_W[i][dir];
double w = neib_ds[dir][1] * KERN_FG_W[i][dir];
sw_side[i][indx] += w;
swd_side[i][indx] += w * neib_ds[dir][0];
}
}
// weights and indices for inner tiles are the same
for (int i = 0; i < KERN_FG.length; i++ ) {
sww_side[i][2] += 1;
sw_side[i][2] += center_str;
swd_side[i][2] += center_str*center_disp;
for (int j = 0; j < 3; j++) {
sw_side2[i][j] = sw_side[i][j];
swd_side2[i][j] = swd_side[i][j];
}
}
for (int dir = 8; dir < 24; dir++) if (neib_ds[dir]!=null){
for (int i = 0; i < KERN_FG.length; i++) if (KERN_FG2[i][dir] > 0){
int indx = KERN_FG2[i][dir] - 1;
double w = neib_ds[dir][1] * KERN_FG_W2[i][dir];
sw_side2[i][indx] += w;
swd_side2[i][indx] += w * neib_ds[dir][0];
}
}
double avg_disparity = Double.NaN;
double avg_strength = Double.NaN;
double avg_center = Double.NaN; // center disparity in the "best" direction
// Maybe replace center_str too?
int best_dir = -1;
// double max_neib_avg = Math.min(center_disp - fsplit_adiff, center_disp * (1.0 - fsplit_rdiff));
for (int i = 0; i < KERN_FG.length; i++) {
if ((num_side[i][0] >= fsplit_neibs_side) && (num_side[i][0] >= fsplit_neibs_side)){
double center_dispm = swd_side[i][2]/sw_side[i][2]; // averaged center disparity
double max_neib_avg = Math.min(center_dispm - fsplit_adiff, center_dispm * (1.0 - fsplit_rdiff));
// center should be above overall average
double avg_all = (swd_side[i][0]+swd_side[i][1]) / (sw_side[i][0] + sw_side[i][1]);
// double diff = center_disp - avg_all;
double [] avg_side = new double[] {swd_side[i][0]/sw_side[i][0], swd_side[i][1]/sw_side[i][1]};
double lowest_near = Math.min(avg_side[0],avg_side[1]);
if ( (lowest_near < max_neib_avg) &&
!(avg_disparity <= lowest_near) && // lower than current best or current is undefined
(avg_all < max_neib_avg)){// overall average is low enough
// Verify that each half (or extended half) is be lower than the center (thin FG is not on a slope)
if ( ((center_dispm > (swd_side[i][0]/sw_side[i][0])) || (center_dispm > (swd_side2[i][0]/sw_side2[i][0]))) &&
((center_dispm > (swd_side[i][1]/sw_side[i][1])) || (center_dispm > (swd_side2[i][1]/sw_side2[i][1])))) {
best_dir = i;
avg_center = center_dispm;
avg_disparity = lowest_near;
avg_strength = (avg_side[1] > avg_side[0])?
( sw_side[i][0]/sww_side[i][0]) :
(sw_side[i][1]/sww_side[i][1]);
}
}
}
}
double disp_diff, kfg;
if (best_dir >= 0) { // found at least one good direction
// for BG use weighted average among near tiles (of 3x3) from the side which is lower
// find BG weighted-average strength
/// double avg_strength = sw_side[best_dir] / skw; // weighted-average strength of the BG
// double avg_disparity = swd_side[best_dir] / sw_side[best_dir];
double disp_fg = avg_center + (avg_center - avg_disparity) * avg_strength / center_str;
disp_diff = disp_fg - avg_disparity;
kfg = (avg_center - avg_disparity) / disp_diff;
// limit kfg
if ((kfg < fsplit_kfg_min) || (kfg > (1.0 - fsplit_kfg_min))) {
kfg = (kfg < fsplit_kfg_min) ? fsplit_kfg_min : (1.0 - fsplit_kfg_min);
disp_fg = (avg_center - avg_disparity)/kfg + avg_disparity;
disp_diff = disp_fg - avg_disparity;
}
far_fgbg[tile] = new double[] {disp_diff, kfg};
sel_out[tile] = true;
if (dbg_thin_fg != null) {
dbg_thin_fg[tile] = true;
}
} else { // not a thin FG over BG, just improve edge
// Keep center_disp at least for now
double sw = 0.0, swd = 0.0;
double n_min = center_disp, n_max = center_disp; // min/max includes center
for (int dir = 0; dir < 8; dir++) if (neib_ds[dir] != null){
sw += neib_ds[dir][1];
swd += neib_ds[dir][1] * neib_ds[dir][0];
num_neibs++;
if (!(n_min <= neib_ds[dir][0])) {
n_min = neib_ds[dir][0];
}
if (!(n_max >= neib_ds[dir][0])) {
n_max = neib_ds[dir][0];
}
}
double davg = swd / sw;
double adiff = Math.abs(center_disp - davg);
// see if it is concave/convex enough
if ((adiff >= fsplit_adiff) && (adiff/davg >= fsplit_rdiff)) {
sel_out[tile] = true;
disp_diff = n_max - n_min;
kfg = (center_disp - n_min) / disp_diff;
if ((kfg < fsplit_kfg_min) || (kfg > (1.0 - fsplit_kfg_min))) {
if (kfg < fsplit_kfg_min) {
n_min = (center_disp - fsplit_kfg_min * n_max) / (1.0 - fsplit_kfg_min);
kfg = fsplit_kfg_min;
} else {
n_max = (center_disp - fsplit_kfg_min * n_min) / (1.0 - fsplit_kfg_min);
kfg = 1.0 - fsplit_kfg_min;
}
disp_diff = n_max - n_min;
}
} else {
continue; // not a thin FG and not concave/convex enough
}
}
far_fgbg[tile] = new double[] {disp_diff, kfg};
sel_out[tile] = true;
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
} else if (split_src == 1) { // target_disparity is FG, selection[] is defined, avg_ds, fg_ds, bg_ds defined
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int tile = ai.getAndIncrement(); tile < nTiles; tile = ai.getAndIncrement()) {
if (selection[tile] && !Double.isNaN(fg_ds[0][tile]) && !Double.isNaN(bg_ds[0][tile])) { // selection should be defined as well as avg_ds and fg_ds
double disp_diff = fg_ds[0][tile] - bg_ds[0][tile];
double kfg = (avg_ds[0][tile] - bg_ds[0][tile])/disp_diff;
if ((kfg >= fsplit_kfg_min) && (kfg <= (1.0 - fsplit_kfg_min))) {
far_fgbg[tile] = new double[] {0.0, kfg};
sel_out[tile] = true;
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
} else if (split_src == 2) { // target_disparity is BG, selection[] is defined, avg_ds, fg_ds, bg_ds defined
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int tile = ai.getAndIncrement(); tile < nTiles; tile = ai.getAndIncrement()) {
if (selection[tile] && !Double.isNaN(fg_ds[0][tile]) && !Double.isNaN(bg_ds[0][tile])) { // selection should be defined as well as avg_ds and fg_ds
double disp_diff = fg_ds[0][tile] - bg_ds[0][tile];
double kfg = (avg_ds[0][tile] - bg_ds[0][tile])/disp_diff;
if ((kfg >= fsplit_kfg_min) && (kfg <= (1.0 - fsplit_kfg_min))) {
far_fgbg[tile] = new double[] {disp_diff, 0.0};
sel_out[tile] = true;
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
}
// TODO: add debug avg_ds[0], fg_ds[0], bg_ds[0], predicted fg_disp, bg_disp, far_fgbg (both), avg_ds[1], fg_ds[1], bg_ds[1],
if (dbg_title != null) {
String [] dbg_titles = {
"avg_disp", // 0
"fg_disp", // 1
"bg_disp", // 2
"fg_pred", // 3
"bg_pred", // 4
"fgbg_diff", // 5
"fgbg_kfg", // 6
"avg_str", // 7
"fg_str", // 8
"bg_str", // 9
"fg_pred_str",//10
"bg_pred_str",//11
"sel_thin"}; //12
double [][] dbg_img = new double [dbg_titles.length][nTiles];
for (int i = 0; i < dbg_img.length; i++) {
Arrays.fill(dbg_img[i], Double.NaN);
}
dbg_img[0] = avg_ds[0];
if (fg_ds != null) dbg_img[1] = fg_ds[0];
if (bg_ds != null) dbg_img[2] = bg_ds[0];
dbg_img[7] = avg_ds[1];
if (fg_ds != null) dbg_img[8] = fg_ds[1];
if (bg_ds != null) dbg_img[9] = bg_ds[1];
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int tile = ai.getAndIncrement(); tile < nTiles; tile = ai.getAndIncrement()) {
dbg_img[12][tile] = (sel_out[tile] ? 0.5 : 0.0) +
(dbg_thin_fg[tile] ? 0.6 : 0.0);
if (far_fgbg[tile] != null) {
double diff = far_fgbg[tile][0];
double kfg = far_fgbg[tile][1];
dbg_img[5][tile] = diff;
dbg_img[6][tile] = kfg;
if (diff == 0) { // target is FG
dbg_img[ 3][tile] = 0.0 + fg_ds[0][tile]; // disparity(FG)
dbg_img[ 4][tile] = (avg_ds[0][tile] - fg_ds[0][tile]) / kfg + fg_ds[0][tile]; // disparity(BG)
dbg_img[10][tile] = avg_ds[1][tile] * kfg; // strength(FG)
dbg_img[11][tile] = avg_ds[1][tile] * (1.0 - kfg); // strength(BG)
} else if (kfg == 0) { // target is BG
dbg_img[ 3][tile] = diff + bg_ds[0][tile]; // disparity(FG)
dbg_img[ 4][tile] = 0.0 + bg_ds[0][tile]; // disparity(BG)
dbg_img[10][tile] = avg_ds[1][tile] * ((avg_ds[0][tile] - bg_ds[0][tile]) / diff); // strength(FG)
dbg_img[11][tile] = avg_ds[1][tile] * ((diff - (avg_ds[0][tile] - bg_ds[0][tile])) / diff); // strength(BG)
} else { // target is unresolved merged maximums
dbg_img[ 3][tile] = 0 + diff * (1.0 - kfg) + avg_ds[0][tile]; // disparity(FG)
dbg_img[ 4][tile] = 0 - diff * kfg + avg_ds[0][tile]; // disparity(BG)
dbg_img[10][tile] = avg_ds[1][tile] * kfg; // strength(FG)
dbg_img[11][tile] = avg_ds[1][tile] * (1.0 - kfg); // strength(BG)
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
ShowDoubleFloatArrays.showArrays(
dbg_img,
tilesX,
tilesY,
true,
dbg_title,
dbg_titles); // dsrbg_titles);
}
return sel_out;
}
public void intersceneMlExport(
CLTParameters clt_parameters,
ErsCorrection ers_reference,
......@@ -8488,6 +9098,7 @@ public class OpticalFlow {
indx_ref, // final int indx_ref,
combo_dsn_final[COMBO_DSN_INDX_DISP], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
null, // final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
margin, // final int margin,
-1, // final int nrefine, // just for debug title
show_2d_corr, // final boolean show_2d_corr,
......@@ -8758,7 +9369,7 @@ public class OpticalFlow {
* @param nSamples number of samples to generate
* @return nSamples-long array of values
*/
public double [] getLappedRandom(
public static double [] getLappedRandom(
double min_exclusive,
double max_exclusive,
int nSamples) {
......@@ -8770,7 +9381,7 @@ public class OpticalFlow {
100);
}
public double [] getLappedRandom(
public static double [] getLappedRandom(
double min_exclusive,
double max_exclusive,
int nSamples,
......@@ -8822,7 +9433,7 @@ public class OpticalFlow {
* @param offsX pixel horizontal (righth) offset to place metadata inside each tile. Typical 0,1,2,3 ...
* @param offsY pixel vertical (down) offset to place metadata inside each tile Typical 15
*/
public void add_tile_meta(
public static void add_tile_meta(
final float [][] fimg,
final int tilesX,
final int tilesY,
......@@ -9012,8 +9623,6 @@ public class OpticalFlow {
}
}
for (int nrefine = 0; nrefine < max_refines; nrefine++) {
/// Runtime.getRuntime().gc();
/// System.out.println("--- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")");
int mcorr_sel = Correlation2d.corrSelEncode(clt_parameters.img_dtt,scenes[indx_ref].getNumSensors());
// FIXME: null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
double [][] disparity_map =
......@@ -9023,6 +9632,7 @@ public class OpticalFlow {
indx_ref, // final int indx_ref,
combo_dsn_change[0], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
null, // final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
margin, // final int margin,
nrefine, // final int nrefine, // just for debug title
( nrefine == (max_refines - 1)) && clt_parameters.inp.show_final_2d, // final boolean show_2d_corr,
......@@ -9031,9 +9641,6 @@ public class OpticalFlow {
false, // final boolean no_map, // do not generate disparity_map (time-consuming LMA)
debug_level-5); // final int debug_level)
/// Runtime.getRuntime().gc();
/// System.out.println("--- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")");
if (debug_level > 0) {
ShowDoubleFloatArrays.showArrays(
disparity_map,
......@@ -9161,7 +9768,7 @@ public class OpticalFlow {
}
public double [][] prepareInitialComboDS(
public static double [][] prepareInitialComboDS(
final CLTParameters clt_parameters,
final QuadCLT [] scenes,
final int indx_ref,
......@@ -9213,6 +9820,7 @@ public class OpticalFlow {
double [] scene_ers_xyz_dt = ers_reference.getSceneErsXYZ_dt(ts);
double [] scene_ers_atr_dt = ers_reference.getSceneErsATR_dt(ts);
initial_toref_ds[i] = getSceneDisparityStrength(
clt_parameters, // final CLTParameters clt_parameters,
true, // final boolean to_ref_disparity, // false - return scene disparity, true - convert disparity back to the reference scene
initial_ds[indx_ref][0], // final double [] disparity_ref, // invalid tiles - NaN in disparity
initial_ds[i][0], // final double [] disparity_scene, // invalid tiles - NaN in disparity (just for masking out invalid scene tiles)
......@@ -9295,6 +9903,7 @@ public class OpticalFlow {
final int indx_ref,
final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid)
final boolean [] selection, // may be null, if not null do not process unselected tiles
final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
final int margin,
final int nrefine, // just for debug title
final boolean show_2d_corr,
......@@ -9332,7 +9941,7 @@ public class OpticalFlow {
}
image_dtt.getCorrelation2d(); // initiate image_dtt.correlation2d, needed if disparity_map != null
double[][] disparity_map = no_map ? null : new double [image_dtt.getDisparityTitles().length][];
final double disparity_corr = 0.00; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
final double disparity_corr = clt_parameters.imp.disparity_corr; // 04/07/2023 // 0.00; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
TpTask[] tp_tasks_ref = null;
//TODO: Need to set scenes[indx_ref].getErsCorrection().setErsDt() before processing scenes ?
......@@ -9464,6 +10073,7 @@ public class OpticalFlow {
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
tp_tasks_ref, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
far_fgbg, // final double [][][] far_fgbg, // null, or [tilesY][tilesX]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
ref_scene.getErsCorrection().getRXY(false), // final double [][] rXY, // from geometryCorrection
// next both can be nulls
null, // final double [][][][] clt_corr_out, // sparse (by the first index) [type][tilesY][tilesX][(2*transform_size-1)*(2*transform_size-1)] or null
......@@ -9523,6 +10133,7 @@ public class OpticalFlow {
image_dtt.clt_process_tl_correlations( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
tp_tasks_ref, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
// only listed tiles will be processed
ref_scene.getErsCorrection().getRXY(false), // final double [][] rXY, // from geometryCorrection
tilesX, // final int tilesX, // tp_tasks may lack maximal tileX, tileY
......@@ -9614,7 +10225,8 @@ public class OpticalFlow {
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
tp_tasks_ref, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
ref_scene.getErsCorrection().getRXY(false), // final double [][] rXY, // from geometryCorrection
far_fgbg, // final double [][] far_fgbg, // null, or [nTile]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
ref_scene.getErsCorrection().getRXY(false), // final double [][] rXY, // from geometryCorrection
// next both can be nulls
null, // final double [][][][] clt_corr_out, // sparse (by the first index) [type][tilesY][tilesX][(2*transform_size-1)*(2*transform_size-1)] or null
// combo will be added as extra pair if mcorr_comb_width > 0 and clt_corr_out has a slot for it
......@@ -9768,7 +10380,7 @@ public class OpticalFlow {
image_dtt.getCorrelation2d(); // initiate image_dtt.correlation2d, needed if disparity_map != null
final double disparity_corr = 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
final double disparity_corr = clt_parameters.imp.disparity_corr; // 04/07/2023 // 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
TpTask[] tp_tasks_ref = null;
double [][] scene_pXpYD;
// transform to self - maybe use a method that sets central points
......@@ -9837,6 +10449,7 @@ public class OpticalFlow {
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
tp_tasks_ref, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
null, // final double [][][] far_fgbg, // null, or [tilesY][tilesX]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
ref_scene.getErsCorrection().getRXY(false), // final double [][] rXY, // from geometryCorrection
// next both can be nulls
null, // final double [][][][] clt_corr_out, // sparse (by the first index) [type][tilesY][tilesX][(2*transform_size-1)*(2*transform_size-1)] or null
......@@ -10342,7 +10955,7 @@ public class OpticalFlow {
return null;
}
return conditionInitialDS(
false, // boolean use_conf, // use configuration parameters, false - use following
true, // 04.07.2023 //false, // boolean use_conf, // use configuration parameters, false - use following
clt_parameters,
dls,
scene,
......@@ -11040,7 +11653,7 @@ public class OpticalFlow {
if (ref_scene.getGPU() != null) {
ref_scene.getGPU().setGpu_debug_level(debug_level - 4); // monitor GPU ops >=-1
}
final double disparity_corr = 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
final double disparity_corr = clt_parameters.imp.disparity_corr; // 04/07/2023 // 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
//ref_disparity
if (ref_disparity == null) {
ref_disparity = ref_scene.getDLS()[use_lma_dsi?1:0];
......@@ -11217,7 +11830,7 @@ public class OpticalFlow {
if (ref_scene.getGPU() != null) {
ref_scene.getGPU().setGpu_debug_level(debug_level - 4); // monitor GPU ops >=-1
}
final double disparity_corr = 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
final double disparity_corr = clt_parameters.imp.disparity_corr; // 04/07/2023 // 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
//ref_disparity
if (ref_disparity == null) {
ref_disparity = ref_scene.getDLS()[use_lma_dsi?1:0];
......
src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java
View file @ 1ddf4e46
......@@ -1394,7 +1394,7 @@ public class QuadCLT extends QuadCLTCPU {
mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, //final double [][] mb_vectors, //
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
0.0, // final double disparity_corr,
clt_parameters.imp.disparity_corr, // 04/07/2023 //0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
threadsMax); // final int threadsMax) // maximal number of threads to launch
......@@ -1407,7 +1407,7 @@ public class QuadCLT extends QuadCLTCPU {
pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
0.0, // final double disparity_corr,
clt_parameters.imp.disparity_corr, // 04/07/2023 // 0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
threadsMax); // final int threadsMax) // maximal number of threads to launch
......@@ -1589,7 +1589,7 @@ public class QuadCLT extends QuadCLTCPU {
mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, //final double [][] mb_vectors, //
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
0.0, // final double disparity_corr,
clt_parameters.imp.disparity_corr, // 04/07/2023 //0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
OpticalFlow.THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
......@@ -1602,7 +1602,7 @@ public class QuadCLT extends QuadCLTCPU {
pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
0.0, // final double disparity_corr,
clt_parameters.imp.disparity_corr, // 04/07/2023 // 0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
OpticalFlow.THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
......@@ -1879,7 +1879,7 @@ public class QuadCLT extends QuadCLTCPU {
mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, //final double [][] mb_vectors, //
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
0.0, // final double disparity_corr,
clt_parameters.imp.disparity_corr, // 04/07/2023 // 0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
OpticalFlow.THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
......@@ -1892,7 +1892,7 @@ public class QuadCLT extends QuadCLTCPU {
pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
0.0, // final double disparity_corr,
clt_parameters.imp.disparity_corr, // 04/07/2023 // 0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
OpticalFlow.THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
......
src/main/java/com/elphel/imagej/tileprocessor/QuadCLTCPU.java
View file @ 1ddf4e46
......@@ -13955,6 +13955,7 @@ public class QuadCLTCPU {
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
tp_tasks, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
null, // final double [][][] far_fgbg, // null, or [tilesY][tilesX]{disp(fg)-disp(bg), str(fg)-str(bg)} hints for LMA FG/BG split
geometryCorrection.getRXY(false), // final double [][] rXY, // from geometryCorrection
// next both can be nulls
clt_corr_out, // null, // final double [][][][] clt_corr_out, // sparse (by the first index) [type][tilesY][tilesX][(2*transform_size-1)*(2*transform_size-1)] or null
......
src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java
View file @ 1ddf4e46
......@@ -9309,7 +9309,7 @@ if (debugLevel > -100) return true; // temporarily !
-1); // debug_level); // int debugLevel)
}
// Does not need to specify reference scene - it is always the last even if out of timestamp order
opticalFlow.intersceneExport(
OpticalFlow.intersceneExport(
clt_parameters, // CLTParameters clt_parameters,
ers_reference, // ErsCorrection ers_reference,
scenes, // QuadCLT [] scenes,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment