diff --git a/src/main/java/com/elphel/imagej/cameras/CLTParameters.java b/src/main/java/com/elphel/imagej/cameras/CLTParameters.java index 889603f49e6834f01aa31ae7fe19060ba15a33d0..3fc8394a3c611d9f953735b9f0d9717d62fd3b6f 100644 --- a/src/main/java/com/elphel/imagej/cameras/CLTParameters.java +++ b/src/main/java/com/elphel/imagej/cameras/CLTParameters.java @@ -163,6 +163,15 @@ public class CLTParameters { public boolean ih_norm_center = true; // Replace samples with a single average with equal weight public boolean inf_restore_disp = true; // Add disparity back to d{x,y}[i] (debug feature) // Lazy eye parameters + + public double ly_gt_strength = 0.18; // use some configurable parameters + public boolean ly_gt_use_wnd = true; + public double ly_gt_rms = 0.2; // split small source samples tp FG/BG if all aux tile RMS exceeds this value + +// boolean split_fg_bg = true; +// boolean for_adjust = false; + + public boolean ly_on_scan = true; // Calculate and apply lazy eye correction after disparity scan (poly or extrinsic) public boolean ly_inf_en = true; // Simultaneously correct disparity at infinity (both poly and extrinsic) public boolean ly_aztilt_en = true; // Adjust azimuths and tilts @@ -922,6 +931,10 @@ public class CLTParameters { properties.setProperty(prefix+"ih_norm_center", this.ih_norm_center+""); properties.setProperty(prefix+"inf_restore_disp", this.inf_restore_disp+""); + properties.setProperty(prefix+"ly_gt_strength", this.ly_gt_strength+""); + properties.setProperty(prefix+"ly_gt_use_wnd", this.ly_gt_use_wnd+""); + properties.setProperty(prefix+"ly_gt_rms", this.ly_gt_rms+""); + properties.setProperty(prefix+"ly_on_scan", this.ly_on_scan+""); properties.setProperty(prefix+"ly_inf_en", this.ly_inf_en+""); properties.setProperty(prefix+"ly_aztilt_en", this.ly_aztilt_en+""); @@ -1627,6 +1640,9 @@ public class CLTParameters { if (properties.getProperty(prefix+"ih_norm_center")!=null) this.ih_norm_center=Boolean.parseBoolean(properties.getProperty(prefix+"ih_norm_center")); if (properties.getProperty(prefix+"inf_restore_disp")!=null) this.inf_restore_disp=Boolean.parseBoolean(properties.getProperty(prefix+"inf_restore_disp")); + if (properties.getProperty(prefix+"ly_gt_strength")!=null) this.ly_gt_strength=Double.parseDouble(properties.getProperty(prefix+"ly_gt_strength")); + if (properties.getProperty(prefix+"ly_gt_use_wnd")!=null) this.ly_gt_use_wnd=Boolean.parseBoolean(properties.getProperty(prefix+"ly_gt_use_wnd")); + if (properties.getProperty(prefix+"ly_gt_rms")!=null) this.ly_gt_rms=Double.parseDouble(properties.getProperty(prefix+"ly_gt_rms")); if (properties.getProperty(prefix+"ly_on_scan")!=null) this.ly_on_scan=Boolean.parseBoolean(properties.getProperty(prefix+"ly_on_scan")); if (properties.getProperty(prefix+"ly_inf_en")!=null) this.ly_inf_en=Boolean.parseBoolean(properties.getProperty(prefix+"ly_inf_en")); if (properties.getProperty(prefix+"ly_aztilt_en")!=null) this.ly_aztilt_en=Boolean.parseBoolean(properties.getProperty(prefix+"ly_aztilt_en")); @@ -2373,12 +2389,17 @@ public class CLTParameters { gd.addTab ("Lazy eye", "Lazy eye parameters"); - gd.addCheckbox ("Calculate and apply lazy eye correction after disparity scan (poly or extrinsic), may repeat", this.ly_on_scan); - gd.addCheckbox ("Adjust disparity using objects at infinity by changing individual tilt and azimuth ", this.ly_inf_en," disable if there are no really far objects in the scene"); + gd.addMessage ("--- main-to-aux depth map parameters ---"); + gd.addNumericField("Minimal reference (main) channel orrelation strength", this.ly_gt_strength, 3); + gd.addCheckbox ("Use window for AUX tiles to reduce weight of the hi-res tiles near low-res tile boundaries", this.ly_gt_use_wnd); + gd.addNumericField("Aux disparity thershold to split FG and BG (and disable AUX tile for adjustment)", this.ly_gt_rms, 3); + gd.addMessage ("--- others ---"); + gd.addCheckbox ("Calculate and apply lazy eye correction after disparity scan (poly or extrinsic), may repeat", this.ly_on_scan); + gd.addCheckbox ("Adjust disparity using objects at infinity by changing individual tilt and azimuth ", this.ly_inf_en," disable if there are no really far objects in the scene"); gd.addCheckbox ("Adjust azimuths and tilts", this.ly_aztilt_en,"Adjust azimuths and tilts excluding those that change disparity"); gd.addCheckbox ("Adjust differential rolls", this.ly_diff_roll_en,"Adjust differential rolls (3 of 4 rolls, keeping average roll)"); gd.addCheckbox ("Correct scales (focal length temperature? variations)", this.ly_focalLength); - gd.addCheckbox ("Enable common roll adjustment (valid for high disparity range scans only)", this.ly_com_roll); + gd.addCheckbox ("Enable common roll adjustment (valid for high disparity range scans only)", this.ly_com_roll); gd.addNumericField("Manual parameter mask selection (0 use checkboxes above)", this.ly_par_sel, 0, 5,"", "bit 0 - sym0, bit1 - sym1, ..."); @@ -3164,7 +3185,6 @@ public class CLTParameters { this.inf_disp_apply= gd.getNextBoolean(); this.inf_repeat= (int) gd.getNextNumber(); - // this.inf_mism_apply= gd.getNextBoolean(); this.inf_iters= (int) gd.getNextNumber(); this.inf_final_diff= gd.getNextNumber(); this.inf_far_pull= gd.getNextNumber(); @@ -3184,6 +3204,9 @@ public class CLTParameters { this.ih_norm_center= gd.getNextBoolean(); this.inf_restore_disp= gd.getNextBoolean(); + this.ly_gt_strength= gd.getNextNumber(); + this.ly_gt_use_wnd= gd.getNextBoolean(); + this.ly_gt_rms= gd.getNextNumber(); this.ly_on_scan= gd.getNextBoolean(); this.ly_inf_en= gd.getNextBoolean(); this.ly_aztilt_en= gd.getNextBoolean(); diff --git a/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java b/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java index 46654a346ca30b0a50954ac9634046a45899709e..46f2fb467f83e6ed4c1864241c0eeaca22a8812a 100644 --- a/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java +++ b/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java @@ -88,7 +88,6 @@ import com.elphel.imagej.lwir.LwirReader; import com.elphel.imagej.readers.EyesisTiff; import com.elphel.imagej.tensorflow.TensorflowInferModel; import com.elphel.imagej.tileprocessor.DttRad2; -import com.elphel.imagej.tileprocessor.GeometryCorrection; import com.elphel.imagej.tileprocessor.ImageDtt; import com.elphel.imagej.tileprocessor.MLStats; import com.elphel.imagej.tileprocessor.QuadCLT; @@ -729,6 +728,7 @@ private Panel panel1, addButton("AUX planes", panelLWIR, color_conf_process_aux); addButton("AUX ASSIGN", panelLWIR, color_process_aux); addButton("AUX OUT 3D", panelLWIR, color_process_aux); + addButton("Main img AUX", panelLWIR, color_process_aux); addButton("Main to AUX", panelLWIR, color_process_aux); addButton("LWIR_TEST", panelLWIR, color_conf_process); @@ -4690,10 +4690,10 @@ private Panel panel1, EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL); clt3d_aux(adjust_extrinsics, adjust_poly); return; - } else if (label.equals("Main to AUX")) { + } else if (label.equals("Main to AUX") || label.equals("Main img AUX")) { DEBUG_LEVEL=MASTER_DEBUG_LEVEL; EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL); - mainToAux(); + mainToAux(label.equals("Main img AUX")); return; // } else if (label.equals("CLT planes")) { @@ -5119,7 +5119,7 @@ private Panel panel1, } /* ======================================================================== */ - public boolean mainToAux() { + public boolean mainToAux(boolean use_img) { if (QUAD_CLT == null){ QUAD_CLT = new QuadCLT ( QuadCLT.PREFIX, @@ -5163,10 +5163,11 @@ private Panel panel1, String [] choices = {"---", "disparity","strength"}; - double min_strength = 0.18; // use some configurable parameters - boolean use_wnd = true; +// double min_strength = 0.18; // use some configurable parameters +// boolean use_wnd = true; boolean split_fg_bg = true; - double split_fbg_rms = 0.2; // split small source samples tp FG/BG if all aux tile RMS exceeds this value +// double split_fbg_rms = 0.2; // split small source samples tp FG/BG if all aux tile RMS exceeds this value + boolean for_adjust = false; GenericDialog gd = new GenericDialog("Select disparity and strength slices"); indx = 0; @@ -5176,10 +5177,12 @@ private Panel panel1, } } gd.addMessage("--- main-to-aux depth map parameters ---"); - gd.addNumericField("Minimal EO correlation strength", min_strength, 3, 6, ""); - gd.addCheckbox("Use window for AUX tiles to reduce weight of the hi-res tiles near low-res tile boundaries" , use_wnd); - gd.addCheckbox("Split FG and BG if hi-res disparity varies for the same low-res tile", split_fg_bg); - gd.addNumericField("Aux disparity thershold to split FG and BG", split_fbg_rms, 3, 6, ""); + gd.addNumericField("Minimal EO correlation strength", CLT_PARAMETERS.ly_gt_strength, 3, 6, ""); + gd.addCheckbox("Use window for AUX tiles to reduce weight of the hi-res tiles near low-res tile boundaries" , CLT_PARAMETERS.ly_gt_use_wnd); + gd.addCheckbox("Split FG and BG if hi-res disparity varies for the same low-res tile", split_fg_bg); + gd.addNumericField("Aux disparity thershold to split FG and BG", CLT_PARAMETERS.ly_gt_rms, 3, 6, ""); + gd.addCheckbox("Data for Lazy Eye correction (remove ambiguous tiles)", for_adjust); + WindowTools.addScrollBars(gd); gd.showDialog(); if (gd.wasCanceled()) return false; @@ -5189,10 +5192,11 @@ private Panel panel1, for (int i = 0; i < selections.length; i++) { selections[i] = gd.getNextChoiceIndex(); } - min_strength = gd.getNextNumber(); - use_wnd = gd.getNextBoolean(); + CLT_PARAMETERS.ly_gt_strength = gd.getNextNumber(); + CLT_PARAMETERS.ly_gt_use_wnd = gd.getNextBoolean(); split_fg_bg = gd.getNextBoolean(); - split_fbg_rms = gd.getNextNumber(); + CLT_PARAMETERS.ly_gt_rms = gd.getNextNumber(); + for_adjust = gd.getNextBoolean(); int index_disparity = -1, index_strength=-1; indx = 0; @@ -5232,177 +5236,24 @@ private Panel panel1, String [] fgbg_titles = {"disparity","strength", "rms","rms-split","fg-disp","fg-str","bg-disp","bg-str"}; String [] rslt_titles = split_fg_bg ? fgbg_titles :titles; - double [][] ds_aux = DepthMapMainToAux( + double [][] ds_aux = QUAD_CLT_AUX.depthMapMainToAux( ds, // double [][] ds, QUAD_CLT.getGeometryCorrection(), // GeometryCorrection geometryCorrection_main, QUAD_CLT_AUX.getGeometryCorrection(), // GeometryCorrection geometryCorrection_aux, CLT_PARAMETERS, - min_strength, // double min_strength, - use_wnd, split_fg_bg, - split_fbg_rms, + for_adjust, DEBUG_LEVEL); // int debug_level (new ShowDoubleFloatArrays()).showArrays(ds_aux, tilesX_aux, tilesY_aux, true, title+"_TOAUX", rslt_titles); + if (ds_aux.length == 2) { + QUAD_CLT_AUX.ds_from_main = ds_aux; + } else { + QUAD_CLT_AUX.ds_from_main = null; + } return true; } - public double [][] DepthMapMainToAux( - double [][] ds, - GeometryCorrection geometryCorrection_main, - GeometryCorrection geometryCorrection_aux, - CLTParameters clt_Parameters, - double min_strength, - boolean use_wnd, - boolean split_fg_bg, - double split_fbg_rms, - int debug_level - ){ - class DS{ - double disparity; // gt disparity - double strength; // gt strength - int tx; // gt tile x - int ty; // gt tile x - double fx; // fractional aux tile X (0.0..1.0) for optional window - double fy; // fractional aux tile Y (0.0..1.0) for optional window -// DS (double disparity, double strength){ -// this.disparity = disparity; -// this.strength = strength; -// } - DS (double disparity, double strength, int tx, int ty, double fx, double fy){ - this.disparity = disparity; - this.strength = strength; - this.tx = tx; - this.ty = ty; - this.fx = fx; - this.fy = fy; - - } - @Override - public String toString() { - return String.format("Disparity (str) = % 6f (%5f), tx=%d ty=%d fx=%5f fy=%5f\n", disparity, strength,tx,ty,fx,fy); - } - } - int tile_size = clt_Parameters.transform_size; - int [] wh_main = geometryCorrection_main.getSensorWH(); - int [] wh_aux = geometryCorrection_aux.getSensorWH(); - int tilesX_main = wh_main[0] / tile_size; - int tilesY_main = wh_main[1] / tile_size; - int tilesX_aux = wh_aux[0] / tile_size; - int tilesY_aux = wh_aux[1] / tile_size; - - ArrayList> ds_list = new ArrayList>(); - for (int nt = 0; nt < tilesX_aux * tilesY_aux; nt++) { - ds_list.add(new ArrayList()); - } - for (int ty = 0; ty < tilesY_main; ty++) { - double centerY = ty * tile_size + tile_size/2; - for (int tx = 0; tx < tilesX_main; tx++) { - int nt = ty*tilesX_main + tx; - double centerX = tx * tile_size + tile_size/2; - double disparity = ds[0][nt]; - double strength = ds[1][nt]; - if ((strength >= min_strength) && !Double.isNaN(disparity)) { - double [] dpxpy_aux = geometryCorrection_aux.getFromOther( - geometryCorrection_main, // GeometryCorrection other_gc, - centerX, // double other_px, - centerY, // double other_py, - disparity); // double other_disparity) - double fx = dpxpy_aux[1]/tile_size; - double fy = dpxpy_aux[2]/tile_size; - int tx_aux = (int) Math.floor(fx); - int ty_aux = (int) Math.floor(fy); - fx -= tx_aux; - fy -= ty_aux; - if ((ty_aux >= 0) && (ty_aux < tilesY_aux) && (tx_aux >= 0) && (tx_aux < tilesX_aux)) { - int nt_aux = ty_aux * tilesX_aux + tx_aux; - ds_list.get(nt_aux).add(new DS(dpxpy_aux[0], strength, tx, ty, fx, fy)); - } - } - } - } - - // simple average (ignoring below minimal) - int num_slices = split_fg_bg? 8:2; - double [][] ds_aux_avg = new double [num_slices][tilesX_aux * tilesY_aux]; - for (int ty = 0; ty < tilesY_aux; ty++) { - for (int tx = 0; tx < tilesX_aux; tx++) { - if ((ty == 4) && (tx == 12)) { - System.out.println("tx = "+tx+", ty = "+ty); - } - int nt = ty * tilesX_aux + tx; - ds_aux_avg[0][nt] = Double.NaN; - ds_aux_avg[1][nt] = 0.0; - if(ds_list.get(nt).isEmpty()) continue; - Collections.sort(ds_list.get(nt), new Comparator() { - @Override - public int compare(DS lhs, DS rhs) { // ascending - return rhs.disparity > lhs.disparity ? -1 : (rhs.disparity < lhs.disparity ) ? 1 : 0; - } - }); - - double sw = 0.0, swd = 0.0, swd2 = 0.0; - for (DS dsi: ds_list.get(nt)) { - double w = dsi.strength; - if (use_wnd) { - w *= Math.sin(Math.PI * (dsi.fx + 0.5/tile_size)) * Math.sin(Math.PI * (dsi.fy + 0.5/tile_size)); - } - sw += w; - double wd = w * dsi.disparity; - swd += wd; - swd2 += wd * dsi.disparity; - } - ds_aux_avg[0][nt] = swd/sw; - ds_aux_avg[1][nt] = sw/ds_list.get(nt).size(); - if (split_fg_bg) { - - ds_aux_avg[2][nt] = Math.sqrt( (swd2 * sw - swd * swd) / (sw * sw)); - ds_aux_avg[3][nt] = ds_aux_avg[2][nt]; // rms - ds_aux_avg[4][nt] = ds_aux_avg[0][nt]; // fg disp - ds_aux_avg[5][nt] = ds_aux_avg[1][nt]; // fg strength - ds_aux_avg[6][nt] = ds_aux_avg[0][nt]; // bg disp - ds_aux_avg[7][nt] = ds_aux_avg[1][nt]; // bg strength - if (ds_aux_avg[2][nt] >= split_fbg_rms) { - // splitting while minimizing sum of 2 squared errors - double [][] swfb = new double [2][ds_list.get(nt).size() -1]; - double [][] swdfb = new double [2][ds_list.get(nt).size() -1]; - double [] s2fb = new double [ds_list.get(nt).size() -1]; - for (int n = 0; n < s2fb.length; n++) { // split position - double [] s2 = new double[2]; - for (int i = 0; i <= s2fb.length; i++) { - int fg = (i > n)? 1 : 0; // 0 - bg, 1 - fg - DS dsi = ds_list.get(nt).get(i); - double w = dsi.strength; - if (use_wnd) { - w *= Math.sin(Math.PI * dsi.fx) * Math.sin(Math.PI * dsi.fy); - } - swfb[fg][n] += w; - double wd = w * dsi.disparity; - swdfb[fg][n] += wd; - s2[fg] += wd * dsi.disparity; - } - s2fb[n] = ((s2[0] * swfb[0][n] - swdfb[0][n] * swdfb[0][n]) / swfb[0][n] + - (s2[1] * swfb[1][n] - swdfb[1][n] * swdfb[1][n]) / swfb[1][n]) / (swfb[0][n] + swfb[1][n]); - } - // now find the n with lowest s2fb and use it to split fg/bg. Could be done in a single pass, but with saved arrays - // it is easier to verify - int nsplit = 0; - for (int i = 1; i < s2fb.length; i++) if (s2fb[i] < s2fb[nsplit]) { - nsplit = i; - } - ds_aux_avg[3][nt] = s2fb[nsplit]; // rms split - ds_aux_avg[4][nt] = swdfb[1][nsplit] / swfb[1][nsplit] ; // fg disp - ds_aux_avg[5][nt] = swfb[1][nsplit]/ (s2fb.length - nsplit) ; // fg strength - - ds_aux_avg[6][nt] = swdfb[0][nsplit] / swfb[0][nsplit] ; // bg disp - ds_aux_avg[7][nt] = swfb[0][nsplit]/ (nsplit + 1) ; // bg strength - } - } - } - } - return ds_aux_avg; - - } //getGeometryCorrection ///data_ssd/lwir3d/results/saved/1562390490_233403/v11/ /* ======================================================================== */ diff --git a/src/main/java/com/elphel/imagej/tileprocessor/AlignmentCorrection.java b/src/main/java/com/elphel/imagej/tileprocessor/AlignmentCorrection.java index d4b88414dfa10252d38f3dde625352d81d3d7c86..477d1a30541d20b4bf732a7b3404eb8e9663bf51 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/AlignmentCorrection.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/AlignmentCorrection.java @@ -2311,7 +2311,7 @@ B = |+dy0 -dy1 -2*dy3 | clt_parameters.ly_par_sel, //int manual_par_sel, // Manually select the parameter mask bit 0 - sym0, bit1 - sym1, ... (0 - use boolean flags, != 0 - ignore boolean flags) mismatch_list, // ArrayList mismatch_list, qc.geometryCorrection, // GeometryCorrection geometryCorrection, - null, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels +/// null, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels qc.geometryCorrection.getCorrVector(), // GeometryCorrection.CorrVector corr_vector, old_new_rms, // double [] old_new_rms, // should be double[2] // 2); // debugLevel); // 2); // 1); // int debugLevel) @@ -2360,7 +2360,7 @@ B = |+dy0 -dy1 -2*dy3 | public double [][][] lazyEyeCorrectionFromGT( - final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig +// final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig final boolean use_poly, // Use polynomial correction, false - correct tilt/azimuth/roll of each sensor final boolean restore_disp_inf, // Restore subtracted disparity for scan #0 (infinity) always true final double fcorr_radius, @@ -2398,7 +2398,6 @@ B = |+dy0 -dy1 -2*dy3 | int debugLevel){ // final double lazyEyeDispRelVariation = 0.02; -//geometryCorrection_main final int dbg_nTile = -34145; // 37005; // -59038; final int num_scans = scans_14.length/NUM_ALL_SLICES; final int num_tiles = scans_14[0].length; @@ -2791,7 +2790,7 @@ B = |+dy0 -dy1 -2*dy3 | clt_parameters.ly_par_sel, // int manual_par_sel, // Manually select the parameter mask bit 0 - sym0, bit1 - sym1, ... (0 - use boolean flags, != 0 - ignore boolean flags) mismatch_list, // ArrayList mismatch_list, qc.geometryCorrection, // GeometryCorrection geometryCorrection, - geometryCorrection_main, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels +/// geometryCorrection_main, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels qc.geometryCorrection.getCorrVector(), // GeometryCorrection.CorrVector corr_vector, old_new_rms, // double [] old_new_rms, // should be double[2] // 2); // debugLevel); // 2); // 1); // int debugLevel) @@ -3326,7 +3325,7 @@ B = |+dy0 -dy1 -2*dy3 | int manual_par_sel, // Manually select the parameter mask bit 0 - sym0, bit1 - sym1, ... (0 - use boolean flags, != 0 - ignore boolean flags) ArrayList mismatch_list, GeometryCorrection geometryCorrection, - GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels +/// GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels GeometryCorrection.CorrVector corr_vector, double [] old_new_rms, // should be double[2] int debugLevel) diff --git a/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java b/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java index 815ab9e9921cdd61ced141712aba762c7cd9c791..654d7b68000ea08bf9ea4007fedb517881a6cfb6 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java @@ -34,6 +34,8 @@ import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Properties; import java.util.Set; @@ -101,7 +103,8 @@ public class QuadCLT { // is enabled double [] lwir_cold_hot = null; // int [] woi_tops; // used to calculate scanline timing - +// just for debugging with the use of intermediate image + public double [][] ds_from_main = null; // magic scale should be set before using TileProcessor (calculated disparities depend on it) public boolean isMonochrome() {return is_mono;} @@ -5999,21 +6002,36 @@ public class QuadCLT { tp.ShowScansSFB( combo_pass_list, // ArrayList scans, // list of composite scans this.image_name+"-SFB"); // String title); -/* - if (show_init_refine) tp.showScan( - combo_pass, // CLTPass3d scan, - "after_compositeScan-"+tp.clt_3d_passes.size()); - - */ - - } if (adjust_extrinsics) { - if (use_rig) { + // temporarily + if (ds_from_main != null) { + System.out.println("Adjust AUX extrinsics using main camera measurements"); + extrinsicsCLTfromGT( +// twoQuadCLT, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + null, + ds_from_main, // gt_disp_strength, + clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters, + adjust_poly, + threadsMax, //final int threadsMax, // maximal number of threads to launch + updateStatus,// final boolean updateStatus, + debugLevel + 2); // final int debugLevel) + + } else if (use_rig) { System.out.println("Adjust extrinsics using rig data here"); + double [][] gt_disp_strength = getRigDSFromTwoQuadCL( + twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + clt_parameters, + debugLevel + 2); // final int debugLevel) + GeometryCorrection geometryCorrection_main = null; + if (geometryCorrection.getRotMatrix(true) != null) { + geometryCorrection_main = twoQuadCLT.quadCLT_main.getGeometryCorrection(); + } extrinsicsCLTfromGT( - twoQuadCLT, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth +// twoQuadCLT, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + geometryCorrection_main, + gt_disp_strength, clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters, adjust_poly, threadsMax, //final int threadsMax, // maximal number of threads to launch @@ -6084,6 +6102,167 @@ public class QuadCLT { IJ.d2s(0.000000001*(System.nanoTime()-this.startTime),3)+" sec, --- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); } + public double [][] depthMapMainToAux( + double [][] ds, + GeometryCorrection geometryCorrection_main, + GeometryCorrection geometryCorrection_aux, + CLTParameters clt_Parameters, +// double min_strength, +// boolean use_wnd, + boolean split_fg_bg, +// double split_fbg_rms, + boolean for_adjust, // for LY adjustment: only keep d,s and remove samples with high variations + int debug_level + ){ + class DS{ + double disparity; // gt disparity + double strength; // gt strength + int tx; // gt tile x + int ty; // gt tile x + double fx; // fractional aux tile X (0.0..1.0) for optional window + double fy; // fractional aux tile Y (0.0..1.0) for optional window + DS (double disparity, double strength, int tx, int ty, double fx, double fy){ + this.disparity = disparity; + this.strength = strength; + this.tx = tx; + this.ty = ty; + this.fx = fx; + this.fy = fy; + + } + @Override + public String toString() { + return String.format("Disparity (str) = % 6f (%5f), tx=%d ty=%d fx=%5f fy=%5f\n", disparity, strength,tx,ty,fx,fy); + } + } + int tile_size = clt_Parameters.transform_size; + int [] wh_main = geometryCorrection_main.getSensorWH(); + int [] wh_aux = geometryCorrection_aux.getSensorWH(); + int tilesX_main = wh_main[0] / tile_size; + int tilesY_main = wh_main[1] / tile_size; + int tilesX_aux = wh_aux[0] / tile_size; + int tilesY_aux = wh_aux[1] / tile_size; + + ArrayList> ds_list = new ArrayList>(); + for (int nt = 0; nt < tilesX_aux * tilesY_aux; nt++) { + ds_list.add(new ArrayList()); + } + for (int ty = 0; ty < tilesY_main; ty++) { + double centerY = ty * tile_size + tile_size/2; + for (int tx = 0; tx < tilesX_main; tx++) { + int nt = ty*tilesX_main + tx; + double centerX = tx * tile_size + tile_size/2; + double disparity = ds[0][nt]; + double strength = ds[1][nt]; + if ((strength >= clt_Parameters.ly_gt_strength) && !Double.isNaN(disparity)) { + double [] dpxpy_aux = geometryCorrection_aux.getFromOther( + geometryCorrection_main, // GeometryCorrection other_gc, + centerX, // double other_px, + centerY, // double other_py, + disparity); // double other_disparity) + double fx = dpxpy_aux[1]/tile_size; + double fy = dpxpy_aux[2]/tile_size; + int tx_aux = (int) Math.floor(fx); + int ty_aux = (int) Math.floor(fy); + fx -= tx_aux; + fy -= ty_aux; + if ((ty_aux >= 0) && (ty_aux < tilesY_aux) && (tx_aux >= 0) && (tx_aux < tilesX_aux)) { + int nt_aux = ty_aux * tilesX_aux + tx_aux; + ds_list.get(nt_aux).add(new DS(dpxpy_aux[0], strength, tx, ty, fx, fy)); + } + } + } + } + + // simple average (ignoring below minimal) + int num_slices = split_fg_bg? 8:2; + double [][] ds_aux_avg = new double [num_slices][tilesX_aux * tilesY_aux]; + for (int ty = 0; ty < tilesY_aux; ty++) { + for (int tx = 0; tx < tilesX_aux; tx++) { + if ((ty == 3) && (tx == 12)) { + System.out.println("tx = "+tx+", ty = "+ty); + } + int nt = ty * tilesX_aux + tx; + ds_aux_avg[0][nt] = Double.NaN; + ds_aux_avg[1][nt] = 0.0; + if(ds_list.get(nt).isEmpty()) continue; + Collections.sort(ds_list.get(nt), new Comparator() { + @Override + public int compare(DS lhs, DS rhs) { // ascending + return rhs.disparity > lhs.disparity ? -1 : (rhs.disparity < lhs.disparity ) ? 1 : 0; + } + }); + + double sw = 0.0, swd = 0.0, swd2 = 0.0; + for (DS dsi: ds_list.get(nt)) { + double w = dsi.strength; + if (clt_Parameters.ly_gt_use_wnd) { + w *= Math.sin(Math.PI * dsi.fx) * Math.sin(Math.PI * dsi.fy); + } + sw += w; + double wd = w * dsi.disparity; + swd += wd; + swd2 += wd * dsi.disparity; + + } + ds_aux_avg[0][nt] = swd/sw; + ds_aux_avg[1][nt] = sw/ds_list.get(nt).size(); + double rms = Math.sqrt( (swd2 * sw - swd * swd) / (sw * sw)); + if (for_adjust && (rms >= clt_Parameters.ly_gt_rms)) { // remove ambiguous tiles + ds_aux_avg[0][nt] = Double.NaN; + ds_aux_avg[1][nt] = 0; + + } + if (split_fg_bg) { + + ds_aux_avg[2][nt] = rms; + ds_aux_avg[3][nt] = ds_aux_avg[2][nt]; // rms + ds_aux_avg[4][nt] = ds_aux_avg[0][nt]; // fg disp + ds_aux_avg[5][nt] = ds_aux_avg[1][nt]; // fg strength + ds_aux_avg[6][nt] = ds_aux_avg[0][nt]; // bg disp + ds_aux_avg[7][nt] = ds_aux_avg[1][nt]; // bg strength + if (rms >= clt_Parameters.ly_gt_rms) { + // splitting while minimizing sum of 2 squared errors + double [][] swfb = new double [2][ds_list.get(nt).size() -1]; + double [][] swdfb = new double [2][ds_list.get(nt).size() -1]; + double [] s2fb = new double [ds_list.get(nt).size() -1]; + for (int n = 0; n < s2fb.length; n++) { // split position + double [] s2 = new double[2]; + for (int i = 0; i <= s2fb.length; i++) { + int fg = (i > n)? 1 : 0; // 0 - bg, 1 - fg + DS dsi = ds_list.get(nt).get(i); + double w = dsi.strength; + if (clt_Parameters.ly_gt_use_wnd) { + w *= Math.sin(Math.PI * dsi.fx) * Math.sin(Math.PI * dsi.fy); + } + swfb[fg][n] += w; + double wd = w * dsi.disparity; + swdfb[fg][n] += wd; + s2[fg] += wd * dsi.disparity; + } + s2fb[n] = ((s2[0] * swfb[0][n] - swdfb[0][n] * swdfb[0][n]) / swfb[0][n] + + (s2[1] * swfb[1][n] - swdfb[1][n] * swdfb[1][n]) / swfb[1][n]) / (swfb[0][n] + swfb[1][n]); + } + // now find the n with lowest s2fb and use it to split fg/bg. Could be done in a single pass, but with saved arrays + // it is easier to verify + int nsplit = 0; + for (int i = 1; i < s2fb.length; i++) if (s2fb[i] < s2fb[nsplit]) { + nsplit = i; + } + ds_aux_avg[3][nt] = s2fb[nsplit]; // rms split + ds_aux_avg[4][nt] = swdfb[1][nsplit] / swfb[1][nsplit] ; // fg disp + ds_aux_avg[5][nt] = swfb[1][nsplit]/ (s2fb.length - nsplit) ; // fg strength + + ds_aux_avg[6][nt] = swdfb[0][nsplit] / swfb[0][nsplit] ; // bg disp + ds_aux_avg[7][nt] = swfb[0][nsplit]/ (nsplit + 1) ; // bg strength + } + } + } + } + return ds_aux_avg; + + } + public boolean preExpandCLTQuad3d( ImagePlus [] imp_quad, // should have properties "name"(base for saving results), "channel","path" boolean [][] saturation_imp, // (near) saturated pixels or null @@ -6903,9 +7082,42 @@ public class QuadCLT { } - public boolean extrinsicsCLTfromGT( + public double [][] getRigDSFromTwoQuadCL( TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth CLTParameters clt_parameters, + final int debugLevel) { + if ((twoQuadCLT == null) || (twoQuadCLT.getBiScan(0) == null)){ + System.out.println("Rig data is not available, aborting"); + return null; + } + + BiScan scan = twoQuadCLT.getBiScan(0); + double [][] rig_disp_strength = scan.getDisparityStrength( + true, // final boolean only_strong, + true, // final boolean only_trusted, + true) ; // final boolean only_enabled); + GeometryCorrection geometryCorrection_main = null; + if (geometryCorrection.getRotMatrix(true) != null) { + geometryCorrection_main = twoQuadCLT.quadCLT_main.getGeometryCorrection(); + double disparityScale = geometryCorrection.getDisparityRadius()/geometryCorrection_main.getDisparityRadius(); + for (int i = 0; i < rig_disp_strength[0].length; i++) { + rig_disp_strength[0][i] *= disparityScale; + } + + if (debugLevel > -2) { + System.out.println("This is an AUX camera, using MAIN camera coordinates"); + } + } + return rig_disp_strength; + + + } + + public boolean extrinsicsCLTfromGT( +// TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + GeometryCorrection geometryCorrection_main, // only used for aux camera if coordinates are for main (null for LWIR) + double [][] rig_disp_strength, + CLTParameters clt_parameters, boolean adjust_poly, final int threadsMax, // maximal number of threads to launch final boolean updateStatus, @@ -6920,6 +7132,7 @@ public class QuadCLT { int max_tries = clt_parameters.lym_iter; // 25; double min_sym_update = clt_parameters.getLymChange(is_aux); // 4e-6; // stop iterations if no angle changes more than this double min_poly_update = clt_parameters.lym_poly_change; // Parameter vector difference to exit from polynomial correction + /* if ((twoQuadCLT == null) || (twoQuadCLT.getBiScan(0) == null)){ System.out.println("Rig data is not available, aborting"); return false; @@ -6951,8 +7164,17 @@ public class QuadCLT { if (debugLevel > -2) { System.out.println("This is an AUX camera, using MAIN camera coordinates"); } - - + } +*/ + if (debugLevel > 20) { + boolean tmp_exit = true; + System.out.println("extrinsicsCLTfromGT()"); + if (tmp_exit) { + System.out.println("will now exit. To continue - change variable tmp_exit in debugger" ); + if (tmp_exit) { + return false; + } + } } CLTPass3d comboScan = tp.compositeScan( @@ -6966,7 +7188,6 @@ public class QuadCLT { // iteration steps double comp_diff = min_sym_update + 1; // (> min_sym_update) for (int num_iter = 0; num_iter < max_tries; num_iter++){ - double [][] combo_mismatch = new double[12][]; CLTMeasure( // perform single pass according to prepared tiles operations and disparity image_data, // first index - number of image in a quad @@ -6998,10 +7219,9 @@ public class QuadCLT { } double [][][] new_corr; -// final double inf_max_disparity = 2.0; double [][][] gt_disparity_strength = {rig_disp_strength}; new_corr = ac.lazyEyeCorrectionFromGT( - geometryCorrection_main, //final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig +/// geometryCorrection_main, //final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig adjust_poly, // final boolean use_poly, true, // final boolean restore_disp_inf, // Restore subtracted disparity for scan #0 (infinity) clt_parameters.fcorr_radius, // final double fcorr_radius, diff --git a/src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java b/src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java index fa5df255c156091b1c77c41fc99ccdc7a47fe8e4..a2edda4534d6fc5142db9a42eac306640239e494 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java @@ -3680,7 +3680,7 @@ if (debugLevel > -100) return true; // temporarily ! // Get DSI from the main camera quadCLT_main.tp.trimCLTPasses(false); // remove rig composite scan if any - // last but not including any rid data + // last but not including any rig data CLTPass3d scan_last = quadCLT_main.tp.clt_3d_passes.get( quadCLT_main.tp.clt_3d_passes_size -1); // get last one double [][] disparity_bimap = setBimapFromCLTPass3d( scan_last, // CLTPass3d scan, @@ -7445,14 +7445,25 @@ if (debugLevel > -100) return true; // temporarily ! System.out.println("Adjusting main camera image set for "+quadCLT_main.image_name+ " (with rig DSI), pass "+(num_adjust_main+1)+" of "+quadCLT_main.correctionsParameters.rig_batch_adjust_main_gt); } + double [][] gt_disp_strength = quadCLT_main.getRigDSFromTwoQuadCL( + this, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + clt_parameters, + debugLevelInner); // final int debugLevel) + +// GeometryCorrection geometryCorrection_main = null; +// if (geometryCorrection.getRotMatrix(true) != null) { +// geometryCorrection_main = twoQuadCLT.quadCLT_main.getGeometryCorrection(); +// } quadCLT_main.extrinsicsCLTfromGT( - this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth - clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters, - false, - threadsMax, //final int threadsMax, // maximal number of threads to launch - updateStatus,// final boolean updateStatus, - debugLevelInner); // final int debugLevel) + // this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + null, + gt_disp_strength, + clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters, + false, + threadsMax, //final int threadsMax, // maximal number of threads to launch + updateStatus,// final boolean updateStatus, + debugLevelInner); // final int debugLevel) } for (int num_adjust_aux = 0; num_adjust_aux < quadCLT_main.correctionsParameters.rig_batch_adjust_aux_gt; num_adjust_aux++) { @@ -7462,9 +7473,15 @@ if (debugLevel > -100) return true; // temporarily ! System.out.println("Adjusting aux camera image set for "+quadCLT_main.image_name+ " (with rig DSI), pass "+(num_adjust_aux+1)+" of "+quadCLT_main.correctionsParameters.rig_batch_adjust_aux_gt); } + double [][] gt_disp_strength = quadCLT_aux.getRigDSFromTwoQuadCL( + this, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + clt_parameters, + debugLevelInner); // final int debugLevel) quadCLT_aux.extrinsicsCLTfromGT( - this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth +// this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth + quadCLT_main.getGeometryCorrection(), + gt_disp_strength, clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters, false, threadsMax, //final int threadsMax, // maximal number of threads to launch