Skip to content

I
imagej-elphel
Commit 0cccb665 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

working on sfm

parent be2cbd56
Hide whitespace changes
Inline Side-by-side
Showing with 416 additions and 140 deletions
src/main/java/com/elphel/imagej/tileprocessor/EstimateSceneRange.java
View file @ 0cccb665
...@@ -272,6 +272,8 @@ public class EstimateSceneRange { ...@@ -272,6 +272,8 @@ public class EstimateSceneRange {
center, earliest, latest, earliest_prescan, latest_prescan, needs_prescan ); center, earliest, latest, earliest_prescan, latest_prescan, needs_prescan );
} }
public static boolean scanSfmIMS( public static boolean scanSfmIMS(
final CLTParameters clt_parameters, final CLTParameters clt_parameters,
EstimateSceneRange estimateSceneRange, EstimateSceneRange estimateSceneRange,
...@@ -285,16 +287,17 @@ public class EstimateSceneRange { ...@@ -285,16 +287,17 @@ public class EstimateSceneRange {
int earliest = estimateSceneRange.earliest; // was earliest_prescan int earliest = estimateSceneRange.earliest; // was earliest_prescan
int latest = estimateSceneRange.latest; // was latest_prescan int latest = estimateSceneRange.latest; // was latest_prescan
// The current version does not use smaller range (prescan) as it uses SFM // The current version does not use smaller range (prescan) as it uses SFM
double extra_max = 1.2; // increase maximal shift in pixels for not-so-accurate estimation (actual was 1.05) // double extra_max = 1.2; // increase maximal shift in pixels for not-so-accurate estimation (actual was 1.05)
RMSEStats rmse_stats = new RMSEStats(); RMSEStats rmse_stats = new RMSEStats();
RMSEStats rmse_stats_metric = clt_parameters.imp.eig_use ? (new RMSEStats()):null; RMSEStats rmse_stats_metric = clt_parameters.imp.eig_use ? (new RMSEStats()):null;
final int cent_index = estimateSceneRange.center; // final int cent_index = estimateSceneRange.center; //
int tilesX = quadCLTs[cent_index].getTileProcessor().getTilesX(); int tilesX = quadCLTs[cent_index].getTileProcessor().getTilesX();
int tilesY = quadCLTs[cent_index].getTileProcessor().getTilesY(); int tilesY = quadCLTs[cent_index].getTileProcessor().getTilesY();
int tile_size = quadCLTs[cent_index].getTileProcessor().getTileSize(); int tiles = tilesX*tilesY;
// int tile_size = quadCLTs[cent_index].getTileProcessor().getTileSize();
boolean lma_use_Z = clt_parameters.imp.lma_use_Z; // true; // lpf x and y, re-adjust X,Y,Z,A,T,R with pull for X,Y. Disables boolean lma_use_Z = clt_parameters.imp.lma_use_Z; // true; // lpf x and y, re-adjust X,Y,Z,A,T,R with pull for X,Y. Disables
boolean lma_use_R = clt_parameters.imp.lma_use_R; // true; // lpf x and y, re-adjust X,Y,Z,A,T,R with pull for X,Y. Disables boolean lma_use_R = clt_parameters.imp.lma_use_R; // true; // lpf x and y, re-adjust X,Y,Z,A,T,R with pull for X,Y. Disables
boolean lock_position = false; boolean lock_position = false;
double min_ref_str = clt_parameters.imp.min_ref_str; double min_ref_str = clt_parameters.imp.min_ref_str;
boolean ref_need_lma = clt_parameters.imp.ref_need_lma; boolean ref_need_lma = clt_parameters.imp.ref_need_lma;
...@@ -310,13 +313,16 @@ public class EstimateSceneRange { ...@@ -310,13 +313,16 @@ public class EstimateSceneRange {
double fmg_distance = clt_parameters.imp.fmg_distance; // try to find other reference scene not closer than this pixels double fmg_distance = clt_parameters.imp.fmg_distance; // try to find other reference scene not closer than this pixels
boolean fmg_rectilinear = clt_parameters.imp.fmg_rectilinear;// use rectilinear model for scene offset estimation boolean fmg_rectilinear = clt_parameters.imp.fmg_rectilinear;// use rectilinear model for scene offset estimation
double min_offset = clt_parameters.imp.min_offset; // 0.0; // double min_offset = clt_parameters.imp.min_offset; // 0.0; //
double max_offset = extra_max * clt_parameters.imp.max_rel_offset * tilesX * tile_size; double max_offset = Double.NaN;// Do not chesk * clt_parameters.imp.max_rel_offset * tilesX * tile_size;
boolean ref_smooth = clt_parameters.imp.ref_smooth; //true; // use SfM filtering if available Not yet available here boolean ref_smooth = clt_parameters.imp.ref_smooth; //true; // use SfM filtering if available Not yet available here
double min_sfm_meters = clt_parameters.imp.min_sfm_meters; // 2.5; // minimal camera offset no use SFM for disparity refinement (~10x camera base) double min_sfm_meters = clt_parameters.imp.min_sfm_meters; // 2.5; // minimal camera offset no use SFM for disparity refinement (~10x camera base)
boolean filt_en = clt_parameters.imp.sfma_filt_en; // true; // Filter disparity during pose adjustment with SfM boolean filt_en = clt_parameters.imp.sfma_filt_en; // true; // Filter disparity during pose adjustment with SfM
boolean filt_each = clt_parameters.imp.sfma_filt_each; // true; // Filter disparity after each step (far enough), false - only once after all Adjustments are over boolean filt_each = clt_parameters.imp.sfma_filt_each; // true; // Filter disparity after each step (far enough), false - only once after all Adjustments are over
double filt_meters = clt_parameters.imp.sfma_filt_meters; // 5.0; // Filter only for high offsets double filt_meters = clt_parameters.imp.sfma_filt_meters; // 5.0; // Filter only for high offsets
boolean sfma_all = clt_parameters.imp.sfma_all_en; // true; // calculate SfM for all tiles, including "unreliable" (make them reliable?)
double sfma_confidence = clt_parameters.imp.sfma_confidence; // 0.8; // rehabilitate "unreliable" SfM tiles if confidence is above this threshold
double [] lma_rms = new double[clt_parameters.imp.eig_use? 6 : 4]; // [2]; double [] lma_rms = new double[clt_parameters.imp.eig_use? 6 : 4]; // [2];
boolean save_reliables = clt_parameters.imp.save_reliables; boolean save_reliables = clt_parameters.imp.save_reliables;
...@@ -392,7 +398,15 @@ public class EstimateSceneRange { ...@@ -392,7 +398,15 @@ public class EstimateSceneRange {
ErsCorrection ers_reference = quadCLTs[cent_index].getErsCorrection(); ErsCorrection ers_reference = quadCLTs[cent_index].getErsCorrection();
// int debug_scene = 57; // -15; last before // int debug_scene = 57; // -15; last before
boolean debug2 = !batch_mode; // false; // true; boolean debug2 = !batch_mode; // false; // true;
/* Will calculate coord_motion for every tile reliable_all masked by too-far to avoid folding
* but use only reliable_ref tiles for orientation. Possibly, recalculate reliable_ref to include
* tiles with good SmF results
*/
boolean [] reliable_ref = null; boolean [] reliable_ref = null;
boolean [] reliable_all = sfma_all ? (new boolean [tiles]) : null;
if (reliable_all!= null) {
Arrays.fill(reliable_all, true);
}
boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi; boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
double [] reduced_strength = new double[1]; double [] reduced_strength = new double[1];
// use combo if second pass? // use combo if second pass?
...@@ -423,9 +437,12 @@ public class EstimateSceneRange { ...@@ -423,9 +437,12 @@ public class EstimateSceneRange {
quadCLTs[cent_index].getTileProcessor().getTilesY(), quadCLTs[cent_index].getTileProcessor().getTilesY(),
quadCLTs[cent_index].getImageName()+"-reliable_ref"); quadCLTs[cent_index].getImageName()+"-reliable_ref");
} }
} else {
reliable_ref = new boolean [tiles];
Arrays.fill(reliable_ref, true);
} }
if (reliables != null) { if (reliables != null) {
reliables[cent_index]=reliable_ref; reliables[cent_index] = reliable_ref;
} }
double [] quat_corr = use_ims_rotation? quadCLTs[cent_index].getQuatCorr() : null; // double [] quat_corr = use_ims_rotation? quadCLTs[cent_index].getQuatCorr() : null; //
double [] enu_corr_metric = quadCLTs[cent_index].getENUCorrMetric(); double [] enu_corr_metric = quadCLTs[cent_index].getENUCorrMetric();
...@@ -464,8 +481,6 @@ public class EstimateSceneRange { ...@@ -464,8 +481,6 @@ public class EstimateSceneRange {
boolean debug_ims_xyzatr = true; boolean debug_ims_xyzatr = true;
if (debug_ims_xyzatr) { if (debug_ims_xyzatr) {
System.out.println(String.format("%6s, %20s, %15s, %15s, %15s, %15s, %15s, %15s", "index","timestamp", "X", "Y", "Z", "Azimuth", "Tilt", "Roll")); System.out.println(String.format("%6s, %20s, %15s, %15s, %15s, %15s, %15s, %15s", "index","timestamp", "X", "Y", "Z", "Azimuth", "Tilt", "Roll"));
// for (int i = 0; i < quadCLTs.length; i++) {
// for (int i = estimateSceneRange.earliest_prescan; i <= estimateSceneRange.latest_prescan ; i++) {
for (int i = earliest; i <= latest ; i++) { for (int i = earliest; i <= latest ; i++) {
System.out.println(String.format("%6d, %20s, %15.12f, %15.12f, %15.12f, %15.12f, %15.12f, %15.12f", i,quadCLTs[i].getImageName(), System.out.println(String.format("%6d, %20s, %15.12f, %15.12f, %15.12f, %15.12f, %15.12f, %15.12f", i,quadCLTs[i].getImageName(),
ims_xyzatr[i][0][0],ims_xyzatr[i][0][1],ims_xyzatr[i][0][2],ims_xyzatr[i][1][0],ims_xyzatr[i][1][1],ims_xyzatr[i][1][2])); ims_xyzatr[i][0][0],ims_xyzatr[i][0][1],ims_xyzatr[i][0][2],ims_xyzatr[i][1][0],ims_xyzatr[i][1][1],ims_xyzatr[i][1][2]));
...@@ -479,7 +494,6 @@ public class EstimateSceneRange { ...@@ -479,7 +494,6 @@ public class EstimateSceneRange {
double [][][] scenes_xyzatr = new double [quadCLTs.length][][]; // previous scene relative to the next one double [][][] scenes_xyzatr = new double [quadCLTs.length][][]; // previous scene relative to the next one
scenes_xyzatr[cent_index] = new double[2][3]; // all zeros scenes_xyzatr[cent_index] = new double[2][3]; // all zeros
// boolean got_spiral = false;
ers_reference.addScene(quadCLTs[cent_index].getImageName(), // add reference scene (itself) too ers_reference.addScene(quadCLTs[cent_index].getImageName(), // add reference scene (itself) too
scenes_xyzatr[cent_index][0], scenes_xyzatr[cent_index][0],
scenes_xyzatr[cent_index][1], scenes_xyzatr[cent_index][1],
...@@ -504,6 +518,8 @@ public class EstimateSceneRange { ...@@ -504,6 +518,8 @@ public class EstimateSceneRange {
boolean img_debug_LMA = debugLevel>1000; // to set manually for debug boolean img_debug_LMA = debugLevel>1000; // to set manually for debug
final double [] sfm_gain2 = new double [tilesX*tilesY]; // sfm gain squared final double [] sfm_gain2 = new double [tilesX*tilesY]; // sfm gain squared
final double [] sfm_confidence = new double [tilesX*tilesY]; // sfm gain squared final double [] sfm_confidence = new double [tilesX*tilesY]; // sfm gain squared
final double [] accum_weights = new double [tilesX*tilesY]; // sfm gain squared
for (int scan_dir = 1; scan_dir >= -1; scan_dir -=2) { for (int scan_dir = 1; scan_dir >= -1; scan_dir -=2) {
...@@ -557,6 +573,10 @@ public class EstimateSceneRange { ...@@ -557,6 +573,10 @@ public class EstimateSceneRange {
boolean [] reliable_scene = quadCLTs[cent_index].maskByOverlap( boolean [] reliable_scene = quadCLTs[cent_index].maskByOverlap(
reliable_ref, // boolean [] reliable_ref_tiles, reliable_ref, // boolean [] reliable_ref_tiles,
center_offset_xy); // double [] offset) center_offset_xy); // double [] offset)
boolean [] reliable_all_scene = quadCLTs[cent_index].maskByOverlap(
reliable_all, // boolean [] reliable_ref_tiles, (null OK)
center_offset_xy); // double [] offset)
if (reliables != null) { if (reliables != null) {
reliables[scene_index]=reliable_scene; reliables[scene_index]=reliable_scene;
} }
...@@ -587,14 +607,15 @@ public class EstimateSceneRange { ...@@ -587,14 +607,15 @@ public class EstimateSceneRange {
min_max, // double [] min_max, // null or pair of minimal and maximal offsets min_max, // double [] min_max, // null or pair of minimal and maximal offsets
fail_reason, // int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max fail_reason, // int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max
quadCLTs, // QuadCLT [] quadCLTs, quadCLTs, // QuadCLT [] quadCLTs,
cent_index, // int ref_index, cent_index, // int ref_index,
tp_tasks_ref, // TpTask[][] tp_tasks_ref, // Should be TpTask[2][*]. If tp_tasks_ref[0] == null will calculate tp_tasks_ref, // TpTask[][] tp_tasks_ref, // Should be TpTask[2][*]. If tp_tasks_ref[0] == null will calculate
// at set first scene to the GPU // at set first scene to the GPU
pXpYD_ref, // double [][] pXpYD_ref, // should be set or at least double [num_tiles][] if tp_tasks_ref[0] == null pXpYD_ref, // double [][] pXpYD_ref, // should be set or at least double [num_tiles][] if tp_tasks_ref[0] == null
cent_index, // int nscene0, // may be == ref_index cent_index, // int nscene0, // may be == ref_index
scene_index, // int nscene1, // compares to nscene0 scene_index, // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity null, // double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
reliable_all_scene, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[cent_index], // double [][] scene0_xyzatr,, scenes_xyzatr[cent_index], // double [][] scene0_xyzatr,,
initial_pose, // double [][] scene1_xyzatr, initial_pose, // double [][] scene1_xyzatr,
...@@ -674,20 +695,37 @@ public class EstimateSceneRange { ...@@ -674,20 +695,37 @@ public class EstimateSceneRange {
System.out.println(); System.out.println();
} }
if (adjust_OK && (coord_motion != null)) { // adjust SFM // compare pXpYD_ref with coord_motion[0] - is it the same? if (adjust_OK && (coord_motion != null)) { // adjust SFM // compare pXpYD_ref with coord_motion[0] - is it the same?
/* Now coord_motionm may have all tiles, not only "reliable" (too far are still excluded in
* reliable_all_scene)
* TODO: use reliable_scene to use by neighbors, update all with with non-null (and strong enough?) correlations.
*
* Rehabilitate all strong enough?
*
*/
boolean sfm_OK = adjustDisparitySFM( // modifies one slice of the quadCLTs[cent_index].dsi boolean sfm_OK = adjustDisparitySFM( // modifies one slice of the quadCLTs[cent_index].dsi
clt_parameters, // final CLTParameters clt_parameters, clt_parameters, // final CLTParameters clt_parameters,
quadCLTs, // final QuadCLT[] quadCLTs, quadCLTs, // final QuadCLT[] quadCLTs,
cent_index, // final int ref_index, cent_index, // final int ref_index,
scene_index, // final int scene_index, scene_index, // final int scene_index,
reliable_scene, // final boolean [] reliable_scene, reliable_scene, // final boolean [] reliable_scene, // updated with new good
reliable_all_scene, // boolean [] reliable_all_scene
coord_motion, // final double [][][] coord_motion, coord_motion, // final double [][][] coord_motion,
sfm_gain2, // final double [] sfm_gain2, // squared sfm_gain2, // final double [] sfm_gain2, // squared
sfm_confidence, // final double [] sfm_confidence, sfm_confidence, // final double [] sfm_confidence,
accum_weights, // final double [] accum_weights,
batch_mode, // final boolean batch_mode, batch_mode, // final boolean batch_mode,
debugLevel); // final int debugLevel) debugLevel); // final int debugLevel)
if (debugLevel > -3) { if (debugLevel > -3) {
System.out.println("Adjusted disparity SFM, result "+sfm_OK+", hor_move = "+hor_move); System.out.println("Adjusted disparity SFM, result "+sfm_OK+", hor_move = "+hor_move);
} }
if (reliable_all_scene != null) {
for (int ntile = 0; ntile < reliable_scene.length; ntile++) {
reliable_ref[ntile] |= reliable_scene[ntile];
}
}
if (filt_en && filt_each && (hor_move >= filt_meters)) { if (filt_en && filt_each && (hor_move >= filt_meters)) {
boolean sfm_filter_OK = FilterDisparitySFM( boolean sfm_filter_OK = FilterDisparitySFM(
clt_parameters, // final CLTParameters clt_parameters, clt_parameters, // final CLTParameters clt_parameters,
...@@ -758,7 +796,7 @@ public class EstimateSceneRange { ...@@ -758,7 +796,7 @@ public class EstimateSceneRange {
use_lma_dsi, // clt_parameters.imp.use_lma_dsi, use_lma_dsi, // clt_parameters.imp.use_lma_dsi,
false, // boolean fpn_disable, // disable fpn filter if images are known to be too close false, // boolean fpn_disable, // disable fpn filter if images are known to be too close
true, // boolean disable_ers, true, // boolean disable_ers,
min_max_fpn, // min_max,// double [] min_max, // null or pair of minimal and maximal offsets min_max_fpn, // min_max,// double [] min_max, // null or pair of minimal and maximal offsets
fail_reason, // int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max fail_reason, // int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max
quadCLTs, // QuadCLT [] quadCLTs, quadCLTs, // QuadCLT [] quadCLTs,
cent_index, // int ref_index, cent_index, // int ref_index,
...@@ -766,13 +804,14 @@ public class EstimateSceneRange { ...@@ -766,13 +804,14 @@ public class EstimateSceneRange {
// at set first scene to the GPU // at set first scene to the GPU
pXpYD_ref, // double [][] pXpYD_ref, // should be set or at least double [num_tiles][] if tp_tasks_ref[0] == null pXpYD_ref, // double [][] pXpYD_ref, // should be set or at least double [num_tiles][] if tp_tasks_ref[0] == null
fpn_pairs[ipair][1], // ref_index, // int nscene0, // may be == ref_index fpn_pairs[ipair][1], // ref_index, // int nscene0, // may be == ref_index
fpn_pairs[ipair][0], // int nscene1, // compares to nscene0 fpn_pairs[ipair][0], // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity null, // double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[fpn_pairs[ipair][1]], // double [][] scene0_xyzatr,, scenes_xyzatr[fpn_pairs[ipair][1]], // double [][] scene0_xyzatr,,
scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [][] scene1_xyzatr, scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [][] scene1_xyzatr,
Double.NaN, // double average_z, Double.NaN, // double average_z,
scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [] scene1_xyzatr_pull, // if both are not null, specify target values to pull to scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [] scene1_xyzatr_pull, // if both are not null, specify target values to pull to
param_select, // clt_parameters.ilp.ilma_lma_select, // boolean[] param_select, param_select, // clt_parameters.ilp.ilma_lma_select, // boolean[] param_select,
reg_weights, // double [] param_regweights, reg_weights, // double [] param_regweights,
...@@ -941,12 +980,14 @@ public class EstimateSceneRange { ...@@ -941,12 +980,14 @@ public class EstimateSceneRange {
public static boolean adjustDisparitySFM( public static boolean adjustDisparitySFM(
final CLTParameters clt_parameters, final CLTParameters clt_parameters,
final QuadCLT[] quadCLTs, final QuadCLT[] quadCLTs,
final int ref_index, final int ref_index, // probably (TODO: verify) non-nulls correspond to reliable_all_scene
final int scene_index, final int scene_index,
final boolean [] reliable, final boolean [] reliable, // used for pose adjustment. Updated with new good!
final boolean [] reliable_all, // evaluate and possibly "rehabilitate" (if not yet reliable) tiles
final double [][][] coord_motion, final double [][][] coord_motion,
final double [] sfm_gain2, // squared. Updated here final double [] sfm_gain2, // squared. Updated here
final double [] sfm_confidence, final double [] sfm_confidence,
final double [] accum_weights,
final boolean batch_mode, final boolean batch_mode,
final int debugLevel) { final int debugLevel) {
boolean debug_src = debugLevel > 1000; boolean debug_src = debugLevel > 1000;
...@@ -957,7 +998,9 @@ public class EstimateSceneRange { ...@@ -957,7 +998,9 @@ public class EstimateSceneRange {
double [][][] scenes_xyzatr = {new double[2][3], ers_reference.getSceneXYZATR(quadCLTs[scene_index].getImageName())}; double [][][] scenes_xyzatr = {new double[2][3], ers_reference.getSceneXYZATR(quadCLTs[scene_index].getImageName())};
double range_disparity_offset = 0; // clt_parameters.imp.range_disparity_offset; double range_disparity_offset = 0; // clt_parameters.imp.range_disparity_offset;
final boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi; final boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
// boolean sfma_all = clt_parameters.imp.sfma_all_en; // true; // calculate SfM for all tiles, including "unreliable" (make them reliable?)
double sfma_confidence = clt_parameters.imp.sfma_confidence; // 0.8; // rehabilitate "unreliable" SfM tiles if confidence is above this threshold
final double flatness_ref = 0.05; // pix
// final boolean filt_en = clt_parameters.imp.sfma_filt_en; // true; // Filter disparity during pose adjustment with SfM // final boolean filt_en = clt_parameters.imp.sfma_filt_en; // true; // Filter disparity during pose adjustment with SfM
// final boolean filt_each = clt_parameters.imp.sfma_filt_each; // true; // Filter disparity after each step (far enough), false - only once after all Adjustments are over // final boolean filt_each = clt_parameters.imp.sfma_filt_each; // true; // Filter disparity after each step (far enough), false - only once after all Adjustments are over
// final double filt_meters = clt_parameters.imp.sfma_filt_meters; // 5.0; // Filter only for high offsets // final double filt_meters = clt_parameters.imp.sfma_filt_meters; // 5.0; // Filter only for high offsets
...@@ -976,40 +1019,117 @@ public class EstimateSceneRange { ...@@ -976,40 +1019,117 @@ public class EstimateSceneRange {
int tiles = tilesX*tilesY; int tiles = tilesX*tilesY;
// double [][] dsi = quadCLTs[ref_index].getDLS(); // double [][] dsi = quadCLTs[ref_index].getDLS();
final double [] ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0]; final double [] ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0];
// Fill NaN here - it will be fast when filled earlier
if (reliable_all != null) { // should be always
/* That means that will process all (considering overlap) tiles, so need to fill NaN disparity
* with some initial approximation. Likely gaps correspond to high (near) objects - hope SfM
* can still work. Maybe start with lower horizontal offsets for such potentially high objects?
* Should be before smoothDisparity()
*/
boolean [] prohibit = reliable_all.clone();
for (int i = 0; i < prohibit.length; i++) {
prohibit[i] = !prohibit[i];
}
double [] ref_disparity1 = TileProcessor.fillNaNs(
ref_disparity, // final double [] data,
prohibit, // final boolean [] prohibit,
tilesX, // int width,
// CAREFUL ! Remaining NaN is grown by unsharp mask filter ************* !
2* tilesX, // 100, // 2*width, // 16, // final int grow,
0.7, // double diagonal_weight, // relative to ortho
100, // int num_passes,
0.03); // final double max_rchange, // = 0.01 - does not need to be accurate
System.arraycopy(ref_disparity1, 0, ref_disparity, 0, ref_disparity.length);
}
double [] ddisp = new double [tiles]; double [] ddisp = new double [tiles];
double [] sfm_gain2_old = debug_src? sfm_gain2.clone() : null; double [] sfm_gain2_old = debug_src? sfm_gain2.clone() : null;
double [] sfm_gain2_new = debug_src? (new double [tiles]) : null; double [] sfm_gain2_new = debug_src? (new double [tiles]) : null;
Arrays.fill(ddisp, Double.NaN); // Arrays.fill(ddisp, Double.NaN);
for (int ntile = 0; ntile < tiles; ntile++) { for (int ntile = 0; ntile < tiles; ntile++) {
double [] dxy = dpXYddisp[ntile]; double [] dxy = dpXYddisp[ntile];
if ((mvect[ntile] != null) && (dxy != null) && reliable[ntile]) { if ((mvect[ntile] != null) && (dxy != null) && reliable_all[ntile]) {
double sfm_g2 = dxy[0]*dxy[0]+dxy[1]*dxy[1]; double sfm_g2 = dxy[0]*dxy[0]+dxy[1]*dxy[1];
if ((sfm_g2 <=0) || Double.isNaN(sfm_g2)) {
System.out.println("adjustDisparitySFM(): sfm_g2="+sfm_g2);
}
if (sfm_gain2_new != null) { if (sfm_gain2_new != null) {
sfm_gain2_new[ntile] = sfm_g2; // for debug images sfm_gain2_new[ntile] = sfm_g2; // for debug images
} }
if (sfm_g2 > sfm_gain2[ntile]) { // only update if the current pair has a larger stereo-base that used // if (sfm_g2 >= sfm_gain2[ntile]) { // only update if the current pair has a larger stereo-base that used
sfm_gain2[ntile] = sfm_g2; /*
sfm_confidence[ntile] = mvect[ntile][2]; * Will use even lower sfm_gain so second half contributions will count too (just according to their weights)
// double sfm_g2_old = sfm_gain[ntile]* */
// seems to be a bug "*" -> "+" double conf = mvect[ntile][2];
/* double dd = -(mvect[ntile][0]*dxy[0] + mvect[ntile][1]*dxy[1]) / sfm_g2;
if ((mvect[ntile] != null) && (dpXYddisp[ntile] != null) && reliable[ntile]) { if (Double.isNaN(dd)) {
double dd = -(mvect[ntile][0]*dpXYddisp[ntile][0] + mvect[ntile][1]*dpXYddisp[ntile][1]) / System.out.println("adjustDisparitySFM(): dd="+dd);
(dpXYddisp[ntile][0]*dpXYddisp[ntile][0] * dpXYddisp[ntile][1] * dpXYddisp[ntile][1]);
*/
double dd = -(mvect[ntile][0]*dxy[0] + mvect[ntile][1]*dxy[1]) / sfm_g2;
ddisp[ntile] = dd;
ref_disparity[ntile] -= dd;
// dsi[0][ntile] -= dd;
// dsi[1][ntile] -= dd;
} }
ddisp[ntile] = dd; // used for debug only, not scaled
double w = conf * sfm_g2;
double w_old = accum_weights[ntile];
double w_new = w_old+w;
accum_weights[ntile] = w_new;
ref_disparity[ntile] -= dd * w/w_new;
sfm_confidence[ntile] = (sfm_confidence[ntile] * w_old + w * conf) / w_new;
sfm_gain2[ntile] = (sfm_gain2[ntile] * w_old + w * sfm_g2) / w_new;
// sfm_gain2[ntile] = sfm_g2; // maximal used
// }
} }
} }
boolean [] reliable_new = (reliable_all != null) ? (new boolean [reliable.length]) : null;
if (reliable_new != null) { // should be always
for (int ntile = 0; ntile < reliable_new.length; ntile++) {
reliable_new[ntile] =
reliable_all[ntile] &&
!reliable[ntile] &&
(mvect[ntile] != null) &&
(mvect[ntile][2] > sfma_confidence);
}
}
if (debug_src) { if (debug_src) {
String [] titles = {"delta-X","delta-Y", "conf","disp","disp-corr","ddisp", "dx/disp","dy/disp", double [] disparity = new double [tiles];
"sfm_gain","sfm_gain_new","sfm_gain_old","pX","pY","reliable"}; for (int ntile = 0; ntile < disparity.length; ntile++) {
disparity[ntile] = (ref_pXpYD[ntile] != null) ? ref_pXpYD[ntile][2] : Double.NaN;
}
double [] curvature2 = getSquaredCurvature(
disparity, // final double [] data,
tilesX); // final int width)
double [] flatness = new double [tiles];
for (int ntile = 0; ntile < disparity.length; ntile++) {
flatness[ntile] = flatness_ref / (flatness_ref + Math.sqrt(curvature2[ntile]));
}
String [] titles = {
"delta-X", // 0
"delta-Y", // 1
"conf", // 2
"disp", // 3
"disp-corr", // 4
"ddisp", // 5
"dx/disp", // 6
"dy/disp", // 7
"sfm_gain", // 8
"sfm_gain_new", // 9
"sfm_gain_old", // 10
"pX", // 11
"pY", // 12
"reliable", // 13
"reliable-all", // 14
"reliable-new", // 15
"conf-reliable", // 16
"conf-unreliable",// 17
"flatness" // 18
};
double [][] dbg_data = new double [titles.length][tiles]; double [][] dbg_data = new double [titles.length][tiles];
for (int i = 0; i < dbg_data.length; i++) { for (int i = 0; i < dbg_data.length; i++) {
Arrays.fill(dbg_data[i], Double.NaN); Arrays.fill(dbg_data[i], Double.NaN);
...@@ -1030,6 +1150,11 @@ public class EstimateSceneRange { ...@@ -1030,6 +1150,11 @@ public class EstimateSceneRange {
dbg_data[ 7][ntile] = dpXYddisp[ntile][1]; dbg_data[ 7][ntile] = dpXYddisp[ntile][1];
} }
dbg_data[13][ntile] = reliable[ntile]? 1.0 : 0.0; dbg_data[13][ntile] = reliable[ntile]? 1.0 : 0.0;
dbg_data[14][ntile] = reliable_all[ntile]? 1.0 : 0.0;
if (reliable_new != null) {
dbg_data[15][ntile] = reliable_new[ntile]? 1.0 : 0.0;
}
dbg_data[ 5][ntile] = ddisp[ntile]; dbg_data[ 5][ntile] = ddisp[ntile];
if ((ref_pXpYD[ntile] != null) && (dpXYddisp[ntile] != null)) { if ((ref_pXpYD[ntile] != null) && (dpXYddisp[ntile] != null)) {
dbg_data[ 4][ntile] = ref_pXpYD[ntile][2] - ddisp[ntile]; dbg_data[ 4][ntile] = ref_pXpYD[ntile][2] - ddisp[ntile];
...@@ -1037,6 +1162,10 @@ public class EstimateSceneRange { ...@@ -1037,6 +1162,10 @@ public class EstimateSceneRange {
dbg_data[ 8][ntile] = Math.sqrt(sfm_gain2[ntile]); dbg_data[ 8][ntile] = Math.sqrt(sfm_gain2[ntile]);
dbg_data[ 9][ntile] = Math.sqrt(sfm_gain2_new[ntile]); dbg_data[ 9][ntile] = Math.sqrt(sfm_gain2_new[ntile]);
dbg_data[10][ntile] = Math.sqrt(sfm_gain2_old[ntile]); dbg_data[10][ntile] = Math.sqrt(sfm_gain2_old[ntile]);
dbg_data[16][ntile] = ( reliable[ntile] && (mvect[ntile] != null) )? mvect[ntile][2] : Double.NaN;
dbg_data[17][ntile] = (!reliable[ntile] && (mvect[ntile] != null) )? mvect[ntile][2] : Double.NaN;
dbg_data[18][ntile] = flatness[ntile];
} }
ShowDoubleFloatArrays.showArrays( // out of boundary 15 ShowDoubleFloatArrays.showArrays( // out of boundary 15
dbg_data, dbg_data,
...@@ -1046,21 +1175,27 @@ public class EstimateSceneRange { ...@@ -1046,21 +1175,27 @@ public class EstimateSceneRange {
scenes[0].getImageName()+"-"+scenes[1].getImageName()+"-test_sfm", scenes[0].getImageName()+"-"+scenes[1].getImageName()+"-test_sfm",
titles); titles);
} }
if (reliable_new != null) {
for (int ntile = 0; ntile < reliable_new.length; ntile++) {
reliable[ntile] |= reliable_new[ntile];
}
}
return true; return true;
} }
//TODO: use confidence from SFM (copied from MV confidence when used for largest sfm gain) as it is now available //TODO: use confidence from SFM (copied from MV confidence when used for largest sfm gain) as it is now available
public static boolean FilterDisparitySFM( public static boolean FilterDisparitySFM(
final CLTParameters clt_parameters, final CLTParameters clt_parameters,
final boolean final_filter, // true after all pose adjustments are finished, fals at intermediate steps // not used in this version, maybe it will later
final boolean final_filter, // true after all pose adjustments are finished, false at intermediate steps
final QuadCLT[] quadCLTs, final QuadCLT[] quadCLTs,
final int ref_index, final int ref_index,
final boolean [] reliable, final boolean [] reliable, // reliable's disparity only updated from other reliable, non-reliable - from all
final int scene_index, // for debug, null _ final final int scene_index, // for debug, null _ final
final boolean batch_mode, final boolean batch_mode,
final int debugLevel) { final int debugLevel) {
boolean gen_img = debugLevel > 1000; boolean gen_img = debugLevel > 1000;
final int dbg_tile = -( 7+23*80); final int dbg_tile = 22+47*80; // ( 7+23*80);
final boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi; final boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
final int filt_iters = clt_parameters.imp.sfma_filt_iters; // 10; // Maximal number of filtering iteration final int filt_iters = clt_parameters.imp.sfma_filt_iters; // 10; // Maximal number of filtering iteration
final int filt_nb_int = clt_parameters.imp.sfma_filt_nb_int; // 6; // minimal number of defined (non-NaN) neighbors for strong internal tiles (excludes 2 columns/rows on each margin) final int filt_nb_int = clt_parameters.imp.sfma_filt_nb_int; // 6; // minimal number of defined (non-NaN) neighbors for strong internal tiles (excludes 2 columns/rows on each margin)
final int filt_nb_edge = clt_parameters.imp.sfma_filt_nb_edge; // 4; // minimal number of defined (non-NaN) neighbors for tiles on the edge of internals (exclude 1 row/col, edges - not neibs) final int filt_nb_edge = clt_parameters.imp.sfma_filt_nb_edge; // 4; // minimal number of defined (non-NaN) neighbors for tiles on the edge of internals (exclude 1 row/col, edges - not neibs)
...@@ -1078,24 +1213,21 @@ public class EstimateSceneRange { ...@@ -1078,24 +1213,21 @@ public class EstimateSceneRange {
final int [] fneib_stats = getEdgeStatus(tilesX, tilesY); final int [] fneib_stats = getEdgeStatus(tilesX, tilesY);
final boolean [] changed = new boolean[tilesX*tilesY]; final boolean [] changed = new boolean[tilesX*tilesY];
final boolean second_min = true; // if old value was not lower than all neighbors, use second minimal disparity for the background, false - use the absolute minimum. final boolean second_min = true; // if old value was not lower than all neighbors, use second minimal disparity for the background, false - use the absolute minimum.
final int min_remain = 2; // remove minimal and maximal if at least this number remains.
final double [][] ref_dbg = gen_img? (new double[filt_iters+1][]) : null; final double [][] ref_dbg = gen_img? (new double[filt_iters+2][]) : null; // 0 - original, last - reliable
if (ref_dbg != null) { if (ref_dbg != null) {
ref_dbg[0] = ref_disparity.clone(); ref_dbg[0] = ref_disparity.clone();
} }
final Thread[] threads = ImageDtt.newThreadArray(); final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0); final AtomicInteger ai = new AtomicInteger(0);
/// final int [] inner_mins = {5,3,1};
final AtomicInteger [] amods = new AtomicInteger[3]; final AtomicInteger [] amods = new AtomicInteger[3];
for (int i = 0; i < amods.length; i++) { for (int i = 0; i < amods.length; i++) {
amods[i] = new AtomicInteger(0); amods[i] = new AtomicInteger(0);
} }
/// int inner_mins_mode = 0;
for (int niter = 0; niter < filt_iters; niter++) { for (int niter = 0; niter < filt_iters; niter++) {
ai.set(0); ai.set(0);
for (AtomicInteger am: amods) am.set(0); for (AtomicInteger am: amods) am.set(0);
/// final int inner_min = inner_mins[inner_mins_mode]; //, then 3,1
for (int ithread = 0; ithread < threads.length; ithread++) { for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() { threads[ithread] = new Thread() {
public void run() { public void run() {
...@@ -1103,33 +1235,65 @@ public class EstimateSceneRange { ...@@ -1103,33 +1235,65 @@ public class EstimateSceneRange {
int [] neib_stat = fneib_stats.clone(); int [] neib_stat = fneib_stats.clone();
for (int nTile = ai.getAndIncrement(); nTile < ref_disparity.length; nTile = ai.getAndIncrement()) { for (int nTile = ai.getAndIncrement(); nTile < ref_disparity.length; nTile = ai.getAndIncrement()) {
if (nTile== dbg_tile) { if (nTile== dbg_tile) {
System.out.println("FilterDisparitySFM(): dbg_tile="+dbg_tile);
} }
if (final_filter || (reliable[nTile])) { // before final only filter calculated boolean center_reliable = reliable[nTile];
//if (final_filter || (reliable[nTile]))
{ // before final only filter calculated
int inner_type = neib_stat[nTile]; int inner_type = neib_stat[nTile];
int min_type = MIN_TYPE[inner_type]; // minimal type for the same distance from the edge int min_type = MIN_TYPE[inner_type]; // minimal type for the same distance from the edge
boolean is_edge = (inner_type - min_type) == 1; boolean is_edge = (inner_type - min_type) == 1;
boolean is_corner = inner_type == min_type; boolean is_corner = inner_type == min_type;
int num_neibs = 0; int num_neibs = 0;
double sum_neibs = 0, sum_neibs2 = 0, min_neibs = Double.NaN, max_neibs = Double.NaN; double sum_neibs = 0, sum_neibs2 = 0, min_neibs = Double.NaN, max_neibs = Double.NaN;
int lowest_dir = -1; // direction of the minimal disparity (to find the second min) int [] outliers_dirs = {-1,-1}; // direction of the minimal and maximal disparity (to find the second min/max)
for (int dir = 0; dir < 8; dir++) { for (int dir = 0; dir < 8; dir++) {
int ntile1 = tn.getNeibIndex(nTile, dir); int ntile1 = tn.getNeibIndex(nTile, dir);
if ((ntile1 >= 0) && (neib_stat[ntile1] >= min_type)) { if ((ntile1 >= 0) && (neib_stat[ntile1] >= min_type)) {
double d = ref_disparity[ntile1]; double d = ref_disparity[ntile1];
if (!Double.isNaN(d)) { /* for unreliable center - any, for reliable - only reliable */
if (!Double.isNaN(d) && (!reliable[nTile] || reliable[ntile1])) {
num_neibs++; num_neibs++;
sum_neibs += d; sum_neibs += d;
sum_neibs2 += d*d; sum_neibs2 += d*d;
if (!(d >= min_neibs)) { if (!(d >= min_neibs)) {
min_neibs = d; min_neibs = d;
lowest_dir = dir; outliers_dirs[0] = dir;
}
if (!(d <= max_neibs)) {
max_neibs = d;
outliers_dirs[1] = dir;
} }
if (!(d <= max_neibs)) max_neibs = d;
} }
} }
} }
// TODO: optimize (first copy to double[8], maybe sort
if ((num_neibs >= filt_nb_int) || (is_edge && (num_neibs >= filt_nb_edge)) || (is_corner && (num_neibs >= filt_nb_corn))){ if ((num_neibs >= filt_nb_int) || (is_edge && (num_neibs >= filt_nb_edge)) || (is_corner && (num_neibs >= filt_nb_corn))){
if (num_neibs >= (min_remain + 2)) { // re-run, removing out
num_neibs = 0;
sum_neibs = 0;
sum_neibs2 = 0;
min_neibs = Double.NaN;
max_neibs = Double.NaN;
for (int dir = 0; dir < 8; dir++) {
int ntile1 = tn.getNeibIndex(nTile, dir);
if ((ntile1 >= 0) && (neib_stat[ntile1] >= min_type) && (dir != outliers_dirs[0]) && (dir != outliers_dirs[1])) {
double d = ref_disparity[ntile1];
/* for unreliable center - any, for reliable - only reliable */
if (!Double.isNaN(d) && (!reliable[nTile] || reliable[ntile1])) {
num_neibs++;
sum_neibs += d;
sum_neibs2 += d*d;
if (!(d >= min_neibs)) {
min_neibs = d;
}
if (!(d <= max_neibs)) {
max_neibs = d;
}
}
}
}
}
// conditions met to modify center tile // conditions met to modify center tile
double avg_disp = sum_neibs/num_neibs; double avg_disp = sum_neibs/num_neibs;
double sigma = Math.sqrt((sum_neibs2- sum_neibs* avg_disp)/(sum_neibs-1)); double sigma = Math.sqrt((sum_neibs2- sum_neibs* avg_disp)/(sum_neibs-1));
...@@ -1141,6 +1305,7 @@ public class EstimateSceneRange { ...@@ -1141,6 +1305,7 @@ public class EstimateSceneRange {
// is it a background tile? NOT using any strengths for now // is it a background tile? NOT using any strengths for now
if (filt_weakbg && ((max_neibs-min_neibs) > filt_fgbg)) { if (filt_weakbg && ((max_neibs-min_neibs) > filt_fgbg)) {
new_d = min_neibs; new_d = min_neibs;
/*
if (second_min && (new_d < old_d)) { if (second_min && (new_d < old_d)) {
double min2_neibs = max_neibs; double min2_neibs = max_neibs;
for (int dir = 0; dir < 8; dir++) if (dir != lowest_dir){ for (int dir = 0; dir < 8; dir++) if (dir != lowest_dir){
...@@ -1155,6 +1320,7 @@ public class EstimateSceneRange { ...@@ -1155,6 +1320,7 @@ public class EstimateSceneRange {
} }
new_d = min2_neibs; new_d = min2_neibs;
} }
*/
} }
ref_disparity_next[nTile] = new_d; ref_disparity_next[nTile] = new_d;
if (!changed[nTile]) { // only count changes to the new tiles if (!changed[nTile]) { // only count changes to the new tiles
...@@ -1197,9 +1363,14 @@ public class EstimateSceneRange { ...@@ -1197,9 +1363,14 @@ public class EstimateSceneRange {
if (ref_dbg != null) { if (ref_dbg != null) {
String [] titles = new String [ref_dbg.length]; String [] titles = new String [ref_dbg.length];
titles[0] = "original"; titles[0] = "original";
for (int i = 1; i < ref_dbg.length; i++) { for (int i = 1; i < ref_dbg.length-1; i++) {
titles[i]="iter-"+(i-1); titles[i]="iter-"+(i-1);
} }
titles[ref_dbg.length-1] = "reliable";
ref_dbg[ref_dbg.length-1] = new double[tilesX*tilesY];
for (int i = 0; i < reliable.length; i++) {
ref_dbg[ref_dbg.length-1][i] = reliable[i] ? 100:Double.NaN;
}
String title = quadCLTs[ref_index].getImageName(); String title = quadCLTs[ref_index].getImageName();
if (scene_index >= 0) { if (scene_index >= 0) {
title += "-"+quadCLTs[scene_index].getImageName(); title += "-"+quadCLTs[scene_index].getImageName();
...@@ -1382,62 +1553,107 @@ public class EstimateSceneRange { ...@@ -1382,62 +1553,107 @@ public class EstimateSceneRange {
return true; return true;
} }
/**
* Get squared difference between each data element and average of its 8 immediate neighbors
* @param data 2d data as a 1d array in a linescan order
* @param width data width (height = data.length/width)
* @return array with the same dimensions as data containing squared differences between the center
* element and average of its neighbors
*/
public static double [] getSquaredCurvature(
final double [] data,
final int width) {
final int height = data.length/width;
final double [] curv = new double [data.length];
final double diag_w = 0.7;
final double [] neib_w = {1.0,diag_w,1.0,diag_w,1.0,diag_w,1.0,diag_w};
Arrays.fill(curv, Double.NaN);
final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
TileNeibs tn = new TileNeibs(width,height);
for (int nTile = ai.getAndIncrement(); nTile < data.length; nTile = ai.getAndIncrement()) {
double d_cent = data[nTile];
if (!Double.isNaN(d_cent)) {
double sumw = 0, sumwd = 0.0;
for (int dir = 0; dir < 8; dir++) {
int neib = tn.getNeibIndex(nTile, dir);
if ((neib >= 0) && !Double.isNaN(data[neib])) {
sumw += neib_w[dir];
sumwd += neib_w[dir] * data[neib];
}
}
if (sumw > 0) {
double diff = d_cent - sumwd/sumw;
curv[nTile] = diff * diff;
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return curv;
}
public static int [] getEdgeStatus( public static int [] getEdgeStatus(
int tilesX, int tilesX,
int tilesY) { int tilesY) {
final int [] edge_stat = new int[tilesX * tilesY]; final int [] edge_stat = new int[tilesX * tilesY];
final Thread[] threads = ImageDtt.newThreadArray(); final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0); final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) { for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() { threads[ithread] = new Thread() {
public void run() { public void run() {
TileNeibs tn = new TileNeibs(tilesX,tilesY); TileNeibs tn = new TileNeibs(tilesX,tilesY);
Rectangle rect_inner = tn.getInnerRectangle(); Rectangle rect_inner = tn.getInnerRectangle();
Rectangle rect_strong_inner = tn.getStrongInnerRectangle(); Rectangle rect_strong_inner = tn.getStrongInnerRectangle();
for (int nTile = ai.getAndIncrement(); nTile < edge_stat.length; nTile = ai.getAndIncrement()) { for (int nTile = ai.getAndIncrement(); nTile < edge_stat.length; nTile = ai.getAndIncrement()) {
Point p = tn.getPoint(nTile); Point p = tn.getPoint(nTile);
if (rect_strong_inner.contains(p)) { if (rect_strong_inner.contains(p)) {
int nn = 0; int nn = 0;
for (int dir = 0; dir < 8; dir++) { for (int dir = 0; dir < 8; dir++) {
if (rect_strong_inner.contains(tn.getPoint(tn.getNeibIndex(nTile, dir)))) nn++; if (rect_strong_inner.contains(tn.getPoint(tn.getNeibIndex(nTile, dir)))) nn++;
} }
switch (nn) { switch (nn) {
case 8: edge_stat[nTile] = TILE_STRONG_INNER; break; case 8: edge_stat[nTile] = TILE_STRONG_INNER; break;
case 5: edge_stat[nTile] = TILE_STRONG_EDGE; break; case 5: edge_stat[nTile] = TILE_STRONG_EDGE; break;
case 3: edge_stat[nTile] = TILE_STRONG_CORNER; break; case 3: edge_stat[nTile] = TILE_STRONG_CORNER; break;
default: default:
throw new IllegalArgumentException ("getEdgeStatus() 1: invalid neighbors: "+nn); throw new IllegalArgumentException ("getEdgeStatus() 1: invalid neighbors: "+nn);
} }
} else if (rect_inner.contains(p)) { } else if (rect_inner.contains(p)) {
int nn = 0; int nn = 0;
for (int dir = 0; dir < 8; dir++) { for (int dir = 0; dir < 8; dir++) {
if (rect_inner.contains(tn.getPoint(tn.getNeibIndex(nTile, dir)))) nn++; if (rect_inner.contains(tn.getPoint(tn.getNeibIndex(nTile, dir)))) nn++;
} }
switch (nn) { switch (nn) {
case 5: edge_stat[nTile] = TILE_INNER_EDGE; break; case 5: edge_stat[nTile] = TILE_INNER_EDGE; break;
case 3: edge_stat[nTile] = TILE_INNER_CORNER; break; case 3: edge_stat[nTile] = TILE_INNER_CORNER; break;
default: default:
throw new IllegalArgumentException ("getEdgeStatus() 2: invalid neighbors: "+nn); throw new IllegalArgumentException ("getEdgeStatus() 2: invalid neighbors: "+nn);
} }
} else { // outer edge } else { // outer edge
int nn = 0; int nn = 0;
for (int dir = 0; dir < 8; dir++) { for (int dir = 0; dir < 8; dir++) {
int ntile1 = tn.getNeibIndex(nTile, dir); int ntile1 = tn.getNeibIndex(nTile, dir);
if (tn.getNeibIndex(nTile, dir) >=0) nn++; if (tn.getNeibIndex(nTile, dir) >=0) nn++;
} }
switch (nn) { switch (nn) {
case 5: edge_stat[nTile] = TILE_EDGE; break; case 5: edge_stat[nTile] = TILE_EDGE; break;
case 3: edge_stat[nTile] = TILE_CORNER; break; case 3: edge_stat[nTile] = TILE_CORNER; break;
default: default:
throw new IllegalArgumentException ("getEdgeStatus() 3: invalid neighbors: "+nn); throw new IllegalArgumentException ("getEdgeStatus() 3: invalid neighbors: "+nn);
} }
} }
} }
} }
}; };
} }
ImageDtt.startAndJoin(threads); ImageDtt.startAndJoin(threads);
return edge_stat; return edge_stat;
} }
......
src/main/java/com/elphel/imagej/tileprocessor/Interscene.java
View file @ 0cccb665
...@@ -872,13 +872,14 @@ public class Interscene { ...@@ -872,13 +872,14 @@ public class Interscene {
ref_index, // int nscene0, // may be == ref_index ref_index, // int nscene0, // may be == ref_index
scene_index, // int nscene1, // compares to nscene0 scene_index, // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity null, // double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[ref_index], // double [][] scene0_xyzatr,, scenes_xyzatr[ref_index], // double [][] scene0_xyzatr,,
initial_pose, // double [][] scene1_xyzatr, initial_pose, // double [][] scene1_xyzatr,
Double.NaN, // double average_z, Double.NaN, // double average_z,
initial_pose, // double [] scene1_xyzatr_pull, // if both are not null, specify target values to pull to initial_pose, // double [] scene1_xyzatr_pull, // if both are not null, specify target values to pull to
param_select, // clt_parameters.ilp.ilma_lma_select, // boolean[] param_select, param_select, // clt_parameters.ilp.ilma_lma_select, // boolean[] param_select,
reg_weights, // double [] param_regweights, reg_weights, // double [] param_regweights,
lma_rms, // double [] rms_out, // null or double [2] lma_rms, // double [] rms_out, // null or double [2]
max_rms, // double max_rms, max_rms, // double max_rms,
...@@ -990,6 +991,7 @@ public class Interscene { ...@@ -990,6 +991,7 @@ public class Interscene {
fpn_pairs[ipair][0], // int nscene1, // compares to nscene0 fpn_pairs[ipair][0], // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity null, // double [] ref_disparity, // null or alternative reference disparity
reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[fpn_pairs[ipair][1]], // double [][] scene0_xyzatr,, scenes_xyzatr[fpn_pairs[ipair][1]], // double [][] scene0_xyzatr,,
scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [][] scene1_xyzatr, scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [][] scene1_xyzatr,
...@@ -1434,7 +1436,8 @@ public class Interscene { ...@@ -1434,7 +1436,8 @@ public class Interscene {
ref_index, // int nscene0, // may be == ref_index ref_index, // int nscene0, // may be == ref_index
scene_index, // int nscene1, // compares to nscene0 scene_index, // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity null, // double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[ref_index], // double [][] scene0_xyzatr,, scenes_xyzatr[ref_index], // double [][] scene0_xyzatr,,
initial_pose, // double [][] scene1_xyzatr, initial_pose, // double [][] scene1_xyzatr,
...@@ -1627,6 +1630,7 @@ public class Interscene { ...@@ -1627,6 +1630,7 @@ public class Interscene {
fpn_pairs[ipair][0], // int nscene1, // compares to nscene0 fpn_pairs[ipair][0], // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity null, // double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true ref_smooth, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[fpn_pairs[ipair][1]], // double [][] scene0_xyzatr,, scenes_xyzatr[fpn_pairs[ipair][1]], // double [][] scene0_xyzatr,,
scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [][] scene1_xyzatr, scenes_xyzatr[fpn_pairs[ipair][0]], // initial_pose, // double [][] scene1_xyzatr,
...@@ -2094,6 +2098,7 @@ public class Interscene { ...@@ -2094,6 +2098,7 @@ public class Interscene {
center_CLT, // QuadCLT first_QuadClt, // First in comparison pair center_CLT, // QuadCLT first_QuadClt, // First in comparison pair
pXpYD_center, // double [][] pXpYD_ref, // pXpYD for the reference scene pXpYD_center, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
tp_tasks_center[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction tp_tasks_center[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
quadCLTs[scene_index], // QuadCLT scene_QuadClt, quadCLTs[scene_index], // QuadCLT scene_QuadClt,
initial_pose, // double [][] camera_xyzatr, initial_pose, // double [][] camera_xyzatr,
...@@ -3439,7 +3444,8 @@ public class Interscene { ...@@ -3439,7 +3444,8 @@ public class Interscene {
test_pair[1], // int nscene1, // compares to nscene0 test_pair[1], // int nscene1, // compares to nscene0
interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[test_pair[0]],// double [][] scene0_xyzatr, scenes_xyzatr[test_pair[0]],// double [][] scene0_xyzatr,
scenes_xyzatr[test_pair[1]], // double [][] scene1_xyzatr, scenes_xyzatr[test_pair[1]], // double [][] scene1_xyzatr,
avg_z, // double average_z, avg_z, // double average_z,
...@@ -3586,8 +3592,9 @@ public class Interscene { ...@@ -3586,8 +3592,9 @@ public class Interscene {
ref_index, // int nscene0, // may be == ref_index ref_index, // int nscene0, // may be == ref_index
nscene, // int nscene1, // compares to nscene0 nscene, // int nscene1, // compares to nscene0
interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[ref_index],// double [][] scene0_xyzatr, scenes_xyzatr[ref_index],// double [][] scene0_xyzatr,
scenes_xyzatr[nscene], // double [][] scene1_xyzatr, scenes_xyzatr[nscene], // double [][] scene1_xyzatr,
avg_z, // double average_z, avg_z, // double average_z,
...@@ -3789,8 +3796,9 @@ public class Interscene { ...@@ -3789,8 +3796,9 @@ public class Interscene {
fpn_pairs[ipair][1], // int nscene0, // may be == ref_index fpn_pairs[ipair][1], // int nscene0, // may be == ref_index
fpn_pairs[ipair][0], // int nscene1, // compares to nscene0 fpn_pairs[ipair][0], // int nscene1, // compares to nscene0
interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity
reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_scene, // reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[fpn_pairs[ipair][1]],// double [][] scene0_xyzatr, scenes_xyzatr[fpn_pairs[ipair][1]],// double [][] scene0_xyzatr,
scenes_xyzatr[fpn_pairs[ipair][0]], // double [][] scene1_xyzatr, scenes_xyzatr[fpn_pairs[ipair][0]], // double [][] scene1_xyzatr,
avg_z, // double average_z, avg_z, // double average_z,
...@@ -3876,7 +3884,8 @@ public class Interscene { ...@@ -3876,7 +3884,8 @@ public class Interscene {
test_pair[0], // int nscene1, // compares to nscene0 test_pair[0], // int nscene1, // compares to nscene0
interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity interscene_ref_disparity,// double [] ref_disparity, // null or alternative reference disparity
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
false, // boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
scenes_xyzatr[test_pair[1]],// double [][] scene0_xyzatr, scenes_xyzatr[test_pair[1]],// double [][] scene0_xyzatr,
scenes_xyzatr[test_pair[0]], // double [][] scene1_xyzatr, scenes_xyzatr[test_pair[0]], // double [][] scene1_xyzatr,
avg_z, // double average_z, avg_z, // double average_z,
...@@ -4426,8 +4435,9 @@ public class Interscene { ...@@ -4426,8 +4435,9 @@ public class Interscene {
center_CLT, // QuadCLT first_QuadClt, // First in comparison pair center_CLT, // QuadCLT first_QuadClt, // First in comparison pair
pXpYD_center, // double [][] pXpYD_ref, // pXpYD for the reference scene pXpYD_center, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
tp_tasks_center[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction tp_tasks_center[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
quadCLTs[nscene], // QuadCLT scene_QuadClt, quadCLTs[nscene], // QuadCLT scene_QuadClt,
initial_pose, // double [][] camera_xyzatr, initial_pose, // double [][] camera_xyzatr,
initial_pose, // double [][] scene_xyzatr_pull, // if both are not null, specify target values to pull to initial_pose, // double [][] scene_xyzatr_pull, // if both are not null, specify target values to pull to
new double[2][3], // double [][] ref_xyzatr, new double[2][3], // double [][] ref_xyzatr,
...@@ -4650,6 +4660,7 @@ public class Interscene { ...@@ -4650,6 +4660,7 @@ public class Interscene {
int nscene1, // compares to nscene0 int nscene1, // compares to nscene0
double [] ref_disparity, // null or alternative reference disparity null double [] ref_disparity, // null or alternative reference disparity null
boolean [] reliable_ref, // null or bitmask of reliable reference tiles boolean [] reliable_ref, // null or bitmask of reliable reference tiles
boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true boolean smooth_disparity, // smooth disparity (according to clt_parameters)+update reliable_ref if true
double [][] scene0_xyzatr, double [][] scene0_xyzatr,
double [][] scene1_xyzatr, double [][] scene1_xyzatr,
...@@ -4666,8 +4677,12 @@ public class Interscene { ...@@ -4666,8 +4677,12 @@ public class Interscene {
boolean img_debug, // generate debug images for each LMA iteration (dangerous, may be many images) boolean img_debug, // generate debug images for each LMA iteration (dangerous, may be many images)
int debugLevel) int debugLevel)
{ {
boolean [] reliable_corr = (reliable_all != null) ? reliable_all : reliable_ref;
double ref_sigma = clt_parameters.imp.ref_sigma; //10.0; // minimal value of the SfM gain maximum to consider available double ref_sigma = clt_parameters.imp.ref_sigma; //10.0; // minimal value of the SfM gain maximum to consider available
double ref_smooth_diff = clt_parameters.imp.ref_smooth_diff; // 0.75; // minimal fraction of the SfM maximal gain double ref_smooth_diff = clt_parameters.imp.ref_smooth_diff; // 0.75; // minimal fraction of the SfM maximal gain
int tilesX = quadCLTs[ref_index].getTileProcessor().getTilesX();
int tilesY = quadCLTs[ref_index].getTileProcessor().getTilesY();
boolean apply_nan = true; boolean apply_nan = true;
if (clt_parameters.imp.ref_smooth_always) { if (clt_parameters.imp.ref_smooth_always) {
smooth_disparity=true; // ********************************************** smooth_disparity=true; // **********************************************
...@@ -4677,12 +4692,34 @@ public class Interscene { ...@@ -4677,12 +4692,34 @@ public class Interscene {
if (ref_disparity == null) { if (ref_disparity == null) {
ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0]; ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0];
} }
boolean debug_disparity = (debugLevel > 2);
if (reliable_all != null) {
/* That means that will process all (considering overlap) tiles, so need to fill NaN disparity
* with some initial approximation. Likely gaps correspond to high (near) objects - hope SfM
* can still work. Maybe start with lower horizontal offsets for such potentially high objects?
* Should be before smoothDisparity()
*/
boolean [] prohibit = reliable_all.clone();
for (int i = 0; i < prohibit.length; i++) {
prohibit[i] = !prohibit[i];
}
ref_disparity = TileProcessor.fillNaNs(
ref_disparity, // final double [] data,
prohibit, // final boolean [] prohibit,
tilesX, // int width,
// CAREFUL ! Remaining NaN is grown by unsharp mask filter ************* !
2* tilesX, // 100, // 2*width, // 16, // final int grow,
0.7, // double diagonal_weight, // relative to ortho
100, // int num_passes,
0.03); // final double max_rchange, // = 0.01 - does not need to be accurate
}
if (smooth_disparity) { if (smooth_disparity) {
double [] dbg_disparity = (debugLevel > 2) ? ref_disparity.clone() : null; double [] dbg_disparity = debug_disparity ? ref_disparity.clone() : null;
boolean [] dbg_reliable = ((dbg_disparity != null) && (reliable_ref != null)) ? reliable_ref.clone(): null; boolean [] dbg_reliable = ((dbg_disparity != null) && (reliable_corr != null)) ? reliable_corr.clone(): null;
ref_disparity = quadCLTs[ref_index].smoothDisparity( ref_disparity = quadCLTs[ref_index].smoothDisparity( // modifies reliable_corr!
ref_disparity, // double [] disparity_in, (will not modify) ref_disparity, // double [] disparity_in, (will not modify)
reliable_ref, // boolean [] reliable_ref, // optional reliable_corr, // boolean [] reliable_ref, // optional
ref_sigma, // double sigma, ref_sigma, // double sigma,
ref_smooth_diff, // double max_diff, ref_smooth_diff, // double max_diff,
apply_nan); // boolean apply_nan) apply_nan); // boolean apply_nan)
...@@ -4699,16 +4736,14 @@ public class Interscene { ...@@ -4699,16 +4736,14 @@ public class Interscene {
if (dbg_reliable != null) { if (dbg_reliable != null) {
dbg_img[2][i] = dbg_reliable[i] ? 10.0: Double.NaN; dbg_img[2][i] = dbg_reliable[i] ? 10.0: Double.NaN;
} }
if (reliable_ref != null) { if (reliable_corr != null) {
dbg_img[3][i] = reliable_ref[i] ? 10.0: Double.NaN; dbg_img[3][i] = reliable_corr[i] ? 10.0: Double.NaN;
} }
} }
int dbg_width = quadCLTs[ref_index].getTileProcessor().getTilesX();
int dbg_height = quadCLTs[ref_index].getTileProcessor().getTilesY();
ShowDoubleFloatArrays.showArrays( ShowDoubleFloatArrays.showArrays(
dbg_img, dbg_img,
dbg_width, tilesX,
dbg_height, tilesY,
true, true,
dbg_title, dbg_title,
dbg_titles); dbg_titles);
...@@ -4765,7 +4800,7 @@ public class Interscene { ...@@ -4765,7 +4800,7 @@ public class Interscene {
ref_disparity,// null // double [] ref_disparity, // null or alternative reference disparity ref_disparity,// null // double [] ref_disparity, // null or alternative reference disparity
pXpYD_scene0, // double [][] ref_pXpYD, pXpYD_scene0, // double [][] ref_pXpYD,
null, // final float [][] fclt, null, // final float [][] fclt,
reliable_ref, // final boolean [] selection, // may be null, if not null do not process unselected tiles reliable_corr, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin, margin, // final int margin,
// motion blur compensation // motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec mb_tau, // double mb_tau, // 0.008; // time constant, sec
...@@ -4850,7 +4885,8 @@ public class Interscene { ...@@ -4850,7 +4885,8 @@ public class Interscene {
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
quadCLTs[nscene0], // QuadCLT reference_QuadClt, quadCLTs[nscene0], // QuadCLT reference_QuadClt,
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // corr, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
reliable_all, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
tp_tasks_ref[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction tp_tasks_ref[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
quadCLTs[nscene1], // QuadCLT scene_QuadClt, quadCLTs[nscene1], // QuadCLT scene_QuadClt,
scene1_xyzatr, // double [][] camera_xyzatr, scene1_xyzatr, // double [][] camera_xyzatr,
...@@ -4919,7 +4955,8 @@ public class Interscene { ...@@ -4919,7 +4955,8 @@ public class Interscene {
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
quadCLTs[nscene0], // QuadCLT first_QuadClt, // First in comparison pair quadCLTs[nscene0], // QuadCLT first_QuadClt, // First in comparison pair
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // corr, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
reliable_all, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
tp_tasks_ref[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction tp_tasks_ref[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
quadCLTs[nscene1], // QuadCLT scene_QuadClt, quadCLTs[nscene1], // QuadCLT scene_QuadClt,
scene1_xyzatr, // double [][] camera_xyzatr, scene1_xyzatr, // double [][] camera_xyzatr,
...@@ -5007,6 +5044,7 @@ public class Interscene { ...@@ -5007,6 +5044,7 @@ public class Interscene {
reference_QuadClt, // QuadCLT first_QuadClt, // First in comparison pair reference_QuadClt, // QuadCLT first_QuadClt, // First in comparison pair
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
null, // boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
tp_tasks_ref2[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction tp_tasks_ref2[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
scene_QuadClt, // QuadCLT scene_QuadClt, scene_QuadClt, // QuadCLT scene_QuadClt,
scene_xyzatr, // double [][] scene_xyzatr, scene_xyzatr, // double [][] scene_xyzatr,
...@@ -5036,6 +5074,7 @@ public class Interscene { ...@@ -5036,6 +5074,7 @@ public class Interscene {
QuadCLT first_QuadClt, // First in comparison pair, does not need to be reference? QuadCLT first_QuadClt, // First in comparison pair, does not need to be reference?
double [][] pXpYD_ref, // pXpYD for the reference scene double [][] pXpYD_ref, // pXpYD for the reference scene
boolean [] reliable_ref, // null or bitmask of reliable reference tiles boolean [] reliable_ref, // null or bitmask of reliable reference tiles
boolean [] reliable_all, // if not null, use to calculate correlations, mask by ref_disparity for pose adjustments
TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
QuadCLT scene_QuadClt, QuadCLT scene_QuadClt,
double [][] scene_xyzatr, double [][] scene_xyzatr,
...@@ -5053,9 +5092,10 @@ public class Interscene { ...@@ -5053,9 +5092,10 @@ public class Interscene {
// boolean img_debug, // generate debug images for each LMA iteration (dangerous, may be many images) // boolean img_debug, // generate debug images for each LMA iteration (dangerous, may be many images)
int debug_level) int debug_level)
{ {
boolean [] reliable_corr = (reliable_all != null) ? reliable_all : reliable_ref;
boolean [] reliable_lma = (reliable_all != null) ? reliable_ref : null; // optionally mask correlations for LMA pose adjustment
boolean dbg_images = false; // show images at each LMA step boolean dbg_images = false; // show images at each LMA step
boolean show_initial = true; // debug feature boolean show_initial = true; // debug feature
// boolean debug_corr_fpn = false;
boolean use3D = clt_parameters.ilp.ilma_3d; boolean use3D = clt_parameters.ilp.ilma_3d;
boolean filter_by_ers = clt_parameters.ilp.ilma_ers_quads; boolean filter_by_ers = clt_parameters.ilp.ilma_ers_quads;
double gap_frac = clt_parameters.ilp.ilma_gap_frac ; // 0.25; double gap_frac = clt_parameters.ilp.ilma_gap_frac ; // 0.25;
...@@ -5074,7 +5114,6 @@ public class Interscene { ...@@ -5074,7 +5114,6 @@ public class Interscene {
double[][] scene_xyzatr0 = new double [][] {scene_xyzatr[0].clone(),scene_xyzatr[1].clone()}; double[][] scene_xyzatr0 = new double [][] {scene_xyzatr[0].clone(),scene_xyzatr[1].clone()};
double [][][] coord_motion = null; double [][][] coord_motion = null;
boolean show_corr_fpn = (!clt_parameters.multiseq_run) && (debug_level > -1); // -3; *********** Change to debug FPN correleation *** boolean show_corr_fpn = (!clt_parameters.multiseq_run) && (debug_level > -1); // -3; *********** Change to debug FPN correleation ***
// float [][] dbg_corr_fpn = debug_corr_fpn ? (new float [34][]) : null;
int nlma = 0; int nlma = 0;
// TODO: save ers_scene.ers_watr_center_dt and ers_scene.ers_wxyz_center_dt before first run lma, restore on failure // TODO: save ers_scene.ers_watr_center_dt and ers_scene.ers_wxyz_center_dt before first run lma, restore on failure
// it is saved to backup parameters when first lma run (nlma == 0), // boolean first_run, // it is saved to backup parameters when first lma run (nlma == 0), // boolean first_run,
...@@ -5097,7 +5136,7 @@ public class Interscene { ...@@ -5097,7 +5136,7 @@ public class Interscene {
scene_QuadClt, // QuadCLT scene_QuadCLT, scene_QuadClt, // QuadCLT scene_QuadCLT,
scene_xyzatr0[0], // xyz scene_xyzatr0[0], // xyz
scene_xyzatr0[1], // pose[1], // atr scene_xyzatr0[1], // pose[1], // atr
reliable_ref, // ****null, // final boolean [] selection, // may be null, if not null do not process unselected tiles reliable_corr, // ****null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin, margin, // final int margin,
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all. sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
facc_2d_img, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs) facc_2d_img, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
...@@ -5186,6 +5225,7 @@ public class Interscene { ...@@ -5186,6 +5225,7 @@ public class Interscene {
eig_min_sqrt, // final double eig_min_sqrt, // 1; // for sqrt(lambda) - consider infinity (infinite linear feature, a line) eig_min_sqrt, // final double eig_min_sqrt, // 1; // for sqrt(lambda) - consider infinity (infinite linear feature, a line)
eigen_masked, // final double [][] eigen, // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1 eigen_masked, // final double [][] eigen, // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
coord_motion[1], // final double [][] vector_XYS, // optical flow X,Y, confidence obtained from the correlate2DIterate() coord_motion[1], // final double [][] vector_XYS, // optical flow X,Y, confidence obtained from the correlate2DIterate()
reliable_lma, // final boolean [] reliable_ref, // additional mask for vector_XYSDS or null
coord_motion[0], // final double [][] centers, // macrotile centers (in pixels and average disparities coord_motion[0], // final double [][] centers, // macrotile centers (in pixels and average disparities
same_weights, // final boolean same_weights, same_weights, // final boolean same_weights,
(nlma == 0), // boolean first_run, (nlma == 0), // boolean first_run,
......
src/main/java/com/elphel/imagej/tileprocessor/IntersceneLma.java
View file @ 0cccb665
...@@ -275,6 +275,7 @@ public class IntersceneLma { ...@@ -275,6 +275,7 @@ public class IntersceneLma {
final double [][] eigen, // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1 final double [][] eigen, // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
// now includes optional Disparity as the last element (for num_components==3) // now includes optional Disparity as the last element (for num_components==3)
final double [][] vector_XYSDS,// optical flow X,Y, confidence obtained from the correlate2DIterate() final double [][] vector_XYSDS,// optical flow X,Y, confidence obtained from the correlate2DIterate()
final boolean [] reliable_ref, // additional mask for vector_XYSDS or null
final double [][] centers, // macrotile centers (in pixels and average disparities final double [][] centers, // macrotile centers (in pixels and average disparities
final boolean same_weights, final boolean same_weights,
boolean first_run, boolean first_run,
...@@ -339,7 +340,8 @@ public class IntersceneLma { ...@@ -339,7 +340,8 @@ public class IntersceneLma {
if (vector_XYSDS != null) {// skip when used for the motion blur vectors, not LMA if (vector_XYSDS != null) {// skip when used for the motion blur vectors, not LMA
setSamplesWeights(vector_XYSDS, // not regularized yet ! // 3d updated setSamplesWeights(vector_XYSDS, // not regularized yet ! // 3d updated
same_weights); // final boolean same_weights) // same weight if > 0 reliable_ref, // final boolean [] reliable_ref, // additional mask for vector_XYSDS or null
same_weights); // final boolean same_weights) // same weight if > 0
} else { } else {
weights = null; // new double[2 * centers.length]; weights = null; // new double[2 * centers.length];
...@@ -482,6 +484,7 @@ public class IntersceneLma { ...@@ -482,6 +484,7 @@ public class IntersceneLma {
if (vector_XYSDS != null) {// skip when used for the motion blur vectors, not LMA if (vector_XYSDS != null) {// skip when used for the motion blur vectors, not LMA
setSamplesWeights(vector_XYSDS, // not regularized yet ! // 3d updated setSamplesWeights(vector_XYSDS, // not regularized yet ! // 3d updated
null, // final boolean [] reliable_ref, // additional mask for vector_XYSDS or null
same_weights); // final boolean same_weights) // same weight if > 0 same_weights); // final boolean same_weights) // same weight if > 0
} else { } else {
weights = null; // new double[2 * centers.length]; weights = null; // new double[2 * centers.length];
...@@ -867,6 +870,7 @@ public class IntersceneLma { ...@@ -867,6 +870,7 @@ public class IntersceneLma {
private void setSamplesWeights( private void setSamplesWeights(
final double [][] vector_XYSDS, // not regularized yet final double [][] vector_XYSDS, // not regularized yet
final boolean [] reliable_ref, // additional mask for vector_XYSDS or null
final boolean same_weights) // same weight if > 0 final boolean same_weights) // same weight if > 0
{ {
//num_components 2 - old, 3 - with disparity //num_components 2 - old, 3 - with disparity
...@@ -883,7 +887,8 @@ public class IntersceneLma { ...@@ -883,7 +887,8 @@ public class IntersceneLma {
threads[ithread] = new Thread() { threads[ithread] = new Thread() {
public void run() { public void run() {
int thread_num = ati.getAndIncrement(); int thread_num = ati.getAndIncrement();
for (int iMTile = ai.getAndIncrement(); iMTile < vector_XYSDS.length; iMTile = ai.getAndIncrement()) if (vector_XYSDS[iMTile] != null){ for (int iMTile = ai.getAndIncrement(); iMTile < vector_XYSDS.length; iMTile = ai.getAndIncrement())
if ((vector_XYSDS[iMTile] != null) && ((reliable_ref == null) || reliable_ref[iMTile])){
double w = vector_XYSDS[iMTile][2]; double w = vector_XYSDS[iMTile][2];
if ((eig_trans != null) && (eig_trans[iMTile] == null)) { if ((eig_trans != null) && (eig_trans[iMTile] == null)) {
w = 0; w = 0;
...@@ -931,7 +936,8 @@ public class IntersceneLma { ...@@ -931,7 +936,8 @@ public class IntersceneLma {
for (int ithread = 0; ithread < threads.length; ithread++) { for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() { threads[ithread] = new Thread() {
public void run() { public void run() {
for (int iMTile = ai.getAndIncrement(); iMTile < vector_XYSDS.length; iMTile = ai.getAndIncrement()) if (vector_XYSDS[iMTile] != null){ for (int iMTile = ai.getAndIncrement(); iMTile < vector_XYSDS.length; iMTile = ai.getAndIncrement())
if ((vector_XYSDS[iMTile] != null) && ((reliable_ref == null) || reliable_ref[iMTile])){
weights[num_components * iMTile] *= s; weights[num_components * iMTile] *= s;
weights[num_components * iMTile + 1] = weights[num_components * iMTile]; weights[num_components * iMTile + 1] = weights[num_components * iMTile];
if (num_components > 2) { if (num_components > 2) {
......
src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java
View file @ 0cccb665
...@@ -158,6 +158,8 @@ public class IntersceneMatchParameters { ...@@ -158,6 +158,8 @@ public class IntersceneMatchParameters {
public double sfma_filt_fgbg = 0.1; // If weak tile's neighbors max-min exceeds this, and public double sfma_filt_fgbg = 0.1; // If weak tile's neighbors max-min exceeds this, and
public double sfma_filt_str = 0.9; // it is NaN or weaker that this fraction of the local maximum, consider it a bg and average with bg tiles only public double sfma_filt_str = 0.9; // it is NaN or weaker that this fraction of the local maximum, consider it a bg and average with bg tiles only
public boolean sfma_all_en = true; // calculate SfM for all tiles, including "unreliable" (make them reliable?)
public double sfma_confidence = 0.15; // rehabilitate "unreliable" SfM tiles if confidence is above this threshold
...@@ -1572,9 +1574,14 @@ min_str_neib_fpn 0.35 ...@@ -1572,9 +1574,14 @@ min_str_neib_fpn 0.35
"Consider weak tiles to belong to bg if the next are met."); "Consider weak tiles to belong to bg if the next are met.");
gd.addNumericField("Min max/min range for BG", this.sfma_filt_fgbg, 6,7,"m", gd.addNumericField("Min max/min range for BG", this.sfma_filt_fgbg, 6,7,"m",
"If weak tile's neighbors max-min exceeds this, and"); "If weak tile's neighbors max-min exceeds this, and");
gd.addNumericField("Maximal BG relative local strength", this.sfma_filt_str, 6,7,"m", gd.addNumericField("Maximal BG relative local strength", this.sfma_filt_str, 6,7,"",
"(continue) it is NaN or weaker that this fraction of the local maximum, consider it a bg and average with bg tiles only"); "(continue) it is NaN or weaker that this fraction of the local maximum, consider it a bg and average with bg tiles only");
gd.addCheckbox ("Calculate SfM for all tiles", this.sfma_all_en,
"Calculate SfM for all tiles, including \"unreliable\", make them reliable (>certain confidence?).");
gd.addNumericField("Rehabilitation confidence", this.sfma_confidence, 6,7,"", // fraction of average "reliable"?
"Interscene correlation confidence to make it \"reliable\".");
//------------------- //-------------------
...@@ -3555,6 +3562,8 @@ min_str_neib_fpn 0.35 ...@@ -3555,6 +3562,8 @@ min_str_neib_fpn 0.35
this.sfma_filt_weakbg = gd.getNextBoolean(); this.sfma_filt_weakbg = gd.getNextBoolean();
this.sfma_filt_fgbg = gd.getNextNumber(); this.sfma_filt_fgbg = gd.getNextNumber();
this.sfma_filt_str = gd.getNextNumber(); this.sfma_filt_str = gd.getNextNumber();
this.sfma_all_en = gd.getNextBoolean();
this.sfma_confidence = gd.getNextNumber();
this.omtch_img_set = ComboMatch.FILES_LISTS_PATHS[gd.getNextChoiceIndex()]; this.omtch_img_set = ComboMatch.FILES_LISTS_PATHS[gd.getNextChoiceIndex()];
this.rln_gpu_width = (int) gd.getNextNumber(); this.rln_gpu_width = (int) gd.getNextNumber();
...@@ -4764,6 +4773,8 @@ min_str_neib_fpn 0.35 ...@@ -4764,6 +4773,8 @@ min_str_neib_fpn 0.35
properties.setProperty(prefix+"sfma_filt_weakbg", this.sfma_filt_weakbg + ""); // boolean properties.setProperty(prefix+"sfma_filt_weakbg", this.sfma_filt_weakbg + ""); // boolean
properties.setProperty(prefix+"sfma_filt_fgbg", this.sfma_filt_fgbg+""); // double properties.setProperty(prefix+"sfma_filt_fgbg", this.sfma_filt_fgbg+""); // double
properties.setProperty(prefix+"sfma_filt_str", this.sfma_filt_str+""); // double properties.setProperty(prefix+"sfma_filt_str", this.sfma_filt_str+""); // double
properties.setProperty(prefix+"sfma_all_en", this.sfma_all_en + ""); // boolean
properties.setProperty(prefix+"sfma_confidence", this.sfma_confidence+""); // double
properties.setProperty(prefix+"omtch_img_set", this.omtch_img_set + ""); // String properties.setProperty(prefix+"omtch_img_set", this.omtch_img_set + ""); // String
properties.setProperty(prefix+"rln_gpu_width", this.rln_gpu_width+""); // int properties.setProperty(prefix+"rln_gpu_width", this.rln_gpu_width+""); // int
...@@ -5916,6 +5927,8 @@ min_str_neib_fpn 0.35 ...@@ -5916,6 +5927,8 @@ min_str_neib_fpn 0.35
if (properties.getProperty(prefix+"sfma_filt_weakbg")!=null) this.sfma_filt_weakbg=Boolean.parseBoolean(properties.getProperty(prefix+"sfma_filt_weakbg")); if (properties.getProperty(prefix+"sfma_filt_weakbg")!=null) this.sfma_filt_weakbg=Boolean.parseBoolean(properties.getProperty(prefix+"sfma_filt_weakbg"));
if (properties.getProperty(prefix+"sfma_filt_fgbg")!=null) this.sfma_filt_fgbg=Double.parseDouble(properties.getProperty(prefix+"sfma_filt_fgbg")); if (properties.getProperty(prefix+"sfma_filt_fgbg")!=null) this.sfma_filt_fgbg=Double.parseDouble(properties.getProperty(prefix+"sfma_filt_fgbg"));
if (properties.getProperty(prefix+"sfma_filt_str")!=null) this.sfma_filt_str=Double.parseDouble(properties.getProperty(prefix+"sfma_filt_str")); if (properties.getProperty(prefix+"sfma_filt_str")!=null) this.sfma_filt_str=Double.parseDouble(properties.getProperty(prefix+"sfma_filt_str"));
if (properties.getProperty(prefix+"sfma_all_en")!=null) this.sfma_all_en=Boolean.parseBoolean(properties.getProperty(prefix+"sfma_all_en"));
if (properties.getProperty(prefix+"sfma_confidence")!=null) this.sfma_confidence=Double.parseDouble(properties.getProperty(prefix+"sfma_confidence"));
if (properties.getProperty(prefix+"omtch_img_set")!=null) this.omtch_img_set=(String) properties.getProperty(prefix+ "omtch_img_set"); if (properties.getProperty(prefix+"omtch_img_set")!=null) this.omtch_img_set=(String) properties.getProperty(prefix+ "omtch_img_set");
if (properties.getProperty(prefix+"rln_gpu_width")!=null) this.rln_gpu_width=Integer.parseInt(properties.getProperty(prefix+"rln_gpu_width")); if (properties.getProperty(prefix+"rln_gpu_width")!=null) this.rln_gpu_width=Integer.parseInt(properties.getProperty(prefix+"rln_gpu_width"));
...@@ -7099,7 +7112,8 @@ min_str_neib_fpn 0.35 ...@@ -7099,7 +7112,8 @@ min_str_neib_fpn 0.35
imp.sfma_filt_weakbg = this.sfma_filt_weakbg; imp.sfma_filt_weakbg = this.sfma_filt_weakbg;
imp.sfma_filt_fgbg = this.sfma_filt_fgbg; imp.sfma_filt_fgbg = this.sfma_filt_fgbg;
imp.sfma_filt_str = this.sfma_filt_str; imp.sfma_filt_str = this.sfma_filt_str;
imp.sfma_all_en = this.sfma_all_en;
imp.sfma_confidence = this.sfma_confidence;
imp.omtch_img_set = this.omtch_img_set; imp.omtch_img_set = this.omtch_img_set;
imp.rln_gpu_width = this.rln_gpu_width; imp.rln_gpu_width = this.rln_gpu_width;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment