Skip to content

I
imagej-elphel
Commit be9da419 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

AUX (LWIR) extrinsic adjustments with main camera ground truth

parent 5075dfa3
Hide whitespace changes
Inline Side-by-side
Showing with 318 additions and 208 deletions
src/main/java/com/elphel/imagej/cameras/CLTParameters.java
View file @ be9da419
......@@ -163,6 +163,15 @@ public class CLTParameters {
public boolean ih_norm_center = true; // Replace samples with a single average with equal weight
public boolean inf_restore_disp = true; // Add disparity back to d{x,y}[i] (debug feature)
// Lazy eye parameters
public double ly_gt_strength = 0.18; // use some configurable parameters
public boolean ly_gt_use_wnd = true;
public double ly_gt_rms = 0.2; // split small source samples tp FG/BG if all aux tile RMS exceeds this value
// boolean split_fg_bg = true;
// boolean for_adjust = false;
public boolean ly_on_scan = true; // Calculate and apply lazy eye correction after disparity scan (poly or extrinsic)
public boolean ly_inf_en = true; // Simultaneously correct disparity at infinity (both poly and extrinsic)
public boolean ly_aztilt_en = true; // Adjust azimuths and tilts
......@@ -922,6 +931,10 @@ public class CLTParameters {
properties.setProperty(prefix+"ih_norm_center", this.ih_norm_center+"");
properties.setProperty(prefix+"inf_restore_disp", this.inf_restore_disp+"");
properties.setProperty(prefix+"ly_gt_strength", this.ly_gt_strength+"");
properties.setProperty(prefix+"ly_gt_use_wnd", this.ly_gt_use_wnd+"");
properties.setProperty(prefix+"ly_gt_rms", this.ly_gt_rms+"");
properties.setProperty(prefix+"ly_on_scan", this.ly_on_scan+"");
properties.setProperty(prefix+"ly_inf_en", this.ly_inf_en+"");
properties.setProperty(prefix+"ly_aztilt_en", this.ly_aztilt_en+"");
......@@ -1627,6 +1640,9 @@ public class CLTParameters {
if (properties.getProperty(prefix+"ih_norm_center")!=null) this.ih_norm_center=Boolean.parseBoolean(properties.getProperty(prefix+"ih_norm_center"));
if (properties.getProperty(prefix+"inf_restore_disp")!=null) this.inf_restore_disp=Boolean.parseBoolean(properties.getProperty(prefix+"inf_restore_disp"));
if (properties.getProperty(prefix+"ly_gt_strength")!=null) this.ly_gt_strength=Double.parseDouble(properties.getProperty(prefix+"ly_gt_strength"));
if (properties.getProperty(prefix+"ly_gt_use_wnd")!=null) this.ly_gt_use_wnd=Boolean.parseBoolean(properties.getProperty(prefix+"ly_gt_use_wnd"));
if (properties.getProperty(prefix+"ly_gt_rms")!=null) this.ly_gt_rms=Double.parseDouble(properties.getProperty(prefix+"ly_gt_rms"));
if (properties.getProperty(prefix+"ly_on_scan")!=null) this.ly_on_scan=Boolean.parseBoolean(properties.getProperty(prefix+"ly_on_scan"));
if (properties.getProperty(prefix+"ly_inf_en")!=null) this.ly_inf_en=Boolean.parseBoolean(properties.getProperty(prefix+"ly_inf_en"));
if (properties.getProperty(prefix+"ly_aztilt_en")!=null) this.ly_aztilt_en=Boolean.parseBoolean(properties.getProperty(prefix+"ly_aztilt_en"));
......@@ -2373,12 +2389,17 @@ public class CLTParameters {
gd.addTab ("Lazy eye", "Lazy eye parameters");
gd.addCheckbox ("Calculate and apply lazy eye correction after disparity scan (poly or extrinsic), may repeat", this.ly_on_scan);
gd.addCheckbox ("Adjust disparity using objects at infinity by changing individual tilt and azimuth ", this.ly_inf_en," disable if there are no really far objects in the scene");
gd.addMessage ("--- main-to-aux depth map parameters ---");
gd.addNumericField("Minimal reference (main) channel orrelation strength", this.ly_gt_strength, 3);
gd.addCheckbox ("Use window for AUX tiles to reduce weight of the hi-res tiles near low-res tile boundaries", this.ly_gt_use_wnd);
gd.addNumericField("Aux disparity thershold to split FG and BG (and disable AUX tile for adjustment)", this.ly_gt_rms, 3);
gd.addMessage ("--- others ---");
gd.addCheckbox ("Calculate and apply lazy eye correction after disparity scan (poly or extrinsic), may repeat", this.ly_on_scan);
gd.addCheckbox ("Adjust disparity using objects at infinity by changing individual tilt and azimuth ", this.ly_inf_en," disable if there are no really far objects in the scene");
gd.addCheckbox ("Adjust azimuths and tilts", this.ly_aztilt_en,"Adjust azimuths and tilts excluding those that change disparity");
gd.addCheckbox ("Adjust differential rolls", this.ly_diff_roll_en,"Adjust differential rolls (3 of 4 rolls, keeping average roll)");
gd.addCheckbox ("Correct scales (focal length temperature? variations)", this.ly_focalLength);
gd.addCheckbox ("Enable common roll adjustment (valid for high disparity range scans only)", this.ly_com_roll);
gd.addCheckbox ("Enable common roll adjustment (valid for high disparity range scans only)", this.ly_com_roll);
gd.addNumericField("Manual parameter mask selection (0 use checkboxes above)", this.ly_par_sel, 0, 5,"",
"bit 0 - sym0, bit1 - sym1, ...");
......@@ -3164,7 +3185,6 @@ public class CLTParameters {
this.inf_disp_apply= gd.getNextBoolean();
this.inf_repeat= (int) gd.getNextNumber();
// this.inf_mism_apply= gd.getNextBoolean();
this.inf_iters= (int) gd.getNextNumber();
this.inf_final_diff= gd.getNextNumber();
this.inf_far_pull= gd.getNextNumber();
......@@ -3184,6 +3204,9 @@ public class CLTParameters {
this.ih_norm_center= gd.getNextBoolean();
this.inf_restore_disp= gd.getNextBoolean();
this.ly_gt_strength= gd.getNextNumber();
this.ly_gt_use_wnd= gd.getNextBoolean();
this.ly_gt_rms= gd.getNextNumber();
this.ly_on_scan= gd.getNextBoolean();
this.ly_inf_en= gd.getNextBoolean();
this.ly_aztilt_en= gd.getNextBoolean();
......
src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java
View file @ be9da419
......@@ -88,7 +88,6 @@ import com.elphel.imagej.lwir.LwirReader;
import com.elphel.imagej.readers.EyesisTiff;
import com.elphel.imagej.tensorflow.TensorflowInferModel;
import com.elphel.imagej.tileprocessor.DttRad2;
import com.elphel.imagej.tileprocessor.GeometryCorrection;
import com.elphel.imagej.tileprocessor.ImageDtt;
import com.elphel.imagej.tileprocessor.MLStats;
import com.elphel.imagej.tileprocessor.QuadCLT;
......@@ -729,6 +728,7 @@ private Panel panel1,
addButton("AUX planes", panelLWIR, color_conf_process_aux);
addButton("AUX ASSIGN", panelLWIR, color_process_aux);
addButton("AUX OUT 3D", panelLWIR, color_process_aux);
addButton("Main img AUX", panelLWIR, color_process_aux);
addButton("Main to AUX", panelLWIR, color_process_aux);
addButton("LWIR_TEST", panelLWIR, color_conf_process);
......@@ -4690,10 +4690,10 @@ private Panel panel1,
EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL);
clt3d_aux(adjust_extrinsics, adjust_poly);
return;
} else if (label.equals("Main to AUX")) {
} else if (label.equals("Main to AUX") || label.equals("Main img AUX")) {
DEBUG_LEVEL=MASTER_DEBUG_LEVEL;
EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL);
mainToAux();
mainToAux(label.equals("Main img AUX"));
return;
//
} else if (label.equals("CLT planes")) {
......@@ -5119,7 +5119,7 @@ private Panel panel1,
}
/* ======================================================================== */
public boolean mainToAux() {
public boolean mainToAux(boolean use_img) {
if (QUAD_CLT == null){
QUAD_CLT = new QuadCLT (
QuadCLT.PREFIX,
......@@ -5163,10 +5163,11 @@ private Panel panel1,
String [] choices = {"---", "disparity","strength"};
double min_strength = 0.18; // use some configurable parameters
boolean use_wnd = true;
// double min_strength = 0.18; // use some configurable parameters
// boolean use_wnd = true;
boolean split_fg_bg = true;
double split_fbg_rms = 0.2; // split small source samples tp FG/BG if all aux tile RMS exceeds this value
// double split_fbg_rms = 0.2; // split small source samples tp FG/BG if all aux tile RMS exceeds this value
boolean for_adjust = false;
GenericDialog gd = new GenericDialog("Select disparity and strength slices");
indx = 0;
......@@ -5176,10 +5177,12 @@ private Panel panel1,
}
}
gd.addMessage("--- main-to-aux depth map parameters ---");
gd.addNumericField("Minimal EO correlation strength", min_strength, 3, 6, "");
gd.addCheckbox("Use window for AUX tiles to reduce weight of the hi-res tiles near low-res tile boundaries" , use_wnd);
gd.addCheckbox("Split FG and BG if hi-res disparity varies for the same low-res tile", split_fg_bg);
gd.addNumericField("Aux disparity thershold to split FG and BG", split_fbg_rms, 3, 6, "");
gd.addNumericField("Minimal EO correlation strength", CLT_PARAMETERS.ly_gt_strength, 3, 6, "");
gd.addCheckbox("Use window for AUX tiles to reduce weight of the hi-res tiles near low-res tile boundaries" , CLT_PARAMETERS.ly_gt_use_wnd);
gd.addCheckbox("Split FG and BG if hi-res disparity varies for the same low-res tile", split_fg_bg);
gd.addNumericField("Aux disparity thershold to split FG and BG", CLT_PARAMETERS.ly_gt_rms, 3, 6, "");
gd.addCheckbox("Data for Lazy Eye correction (remove ambiguous tiles)", for_adjust);
WindowTools.addScrollBars(gd);
gd.showDialog();
if (gd.wasCanceled()) return false;
......@@ -5189,10 +5192,11 @@ private Panel panel1,
for (int i = 0; i < selections.length; i++) {
selections[i] = gd.getNextChoiceIndex();
}
min_strength = gd.getNextNumber();
use_wnd = gd.getNextBoolean();
CLT_PARAMETERS.ly_gt_strength = gd.getNextNumber();
CLT_PARAMETERS.ly_gt_use_wnd = gd.getNextBoolean();
split_fg_bg = gd.getNextBoolean();
split_fbg_rms = gd.getNextNumber();
CLT_PARAMETERS.ly_gt_rms = gd.getNextNumber();
for_adjust = gd.getNextBoolean();
int index_disparity = -1, index_strength=-1;
indx = 0;
......@@ -5232,177 +5236,24 @@ private Panel panel1,
String [] fgbg_titles = {"disparity","strength", "rms","rms-split","fg-disp","fg-str","bg-disp","bg-str"};
String [] rslt_titles = split_fg_bg ? fgbg_titles :titles;
double [][] ds_aux = DepthMapMainToAux(
double [][] ds_aux = QUAD_CLT_AUX.depthMapMainToAux(
ds, // double [][] ds,
QUAD_CLT.getGeometryCorrection(), // GeometryCorrection geometryCorrection_main,
QUAD_CLT_AUX.getGeometryCorrection(), // GeometryCorrection geometryCorrection_aux,
CLT_PARAMETERS,
min_strength, // double min_strength,
use_wnd,
split_fg_bg,
split_fbg_rms,
for_adjust,
DEBUG_LEVEL); // int debug_level
(new ShowDoubleFloatArrays()).showArrays(ds_aux, tilesX_aux, tilesY_aux, true, title+"_TOAUX", rslt_titles);
if (ds_aux.length == 2) {
QUAD_CLT_AUX.ds_from_main = ds_aux;
} else {
QUAD_CLT_AUX.ds_from_main = null;
}
return true;
}
public double [][] DepthMapMainToAux(
double [][] ds,
GeometryCorrection geometryCorrection_main,
GeometryCorrection geometryCorrection_aux,
CLTParameters clt_Parameters,
double min_strength,
boolean use_wnd,
boolean split_fg_bg,
double split_fbg_rms,
int debug_level
){
class DS{
double disparity; // gt disparity
double strength; // gt strength
int tx; // gt tile x
int ty; // gt tile x
double fx; // fractional aux tile X (0.0..1.0) for optional window
double fy; // fractional aux tile Y (0.0..1.0) for optional window
// DS (double disparity, double strength){
// this.disparity = disparity;
// this.strength = strength;
// }
DS (double disparity, double strength, int tx, int ty, double fx, double fy){
this.disparity = disparity;
this.strength = strength;
this.tx = tx;
this.ty = ty;
this.fx = fx;
this.fy = fy;
}
@Override
public String toString() {
return String.format("Disparity (str) = % 6f (%5f), tx=%d ty=%d fx=%5f fy=%5f\n", disparity, strength,tx,ty,fx,fy);
}
}
int tile_size = clt_Parameters.transform_size;
int [] wh_main = geometryCorrection_main.getSensorWH();
int [] wh_aux = geometryCorrection_aux.getSensorWH();
int tilesX_main = wh_main[0] / tile_size;
int tilesY_main = wh_main[1] / tile_size;
int tilesX_aux = wh_aux[0] / tile_size;
int tilesY_aux = wh_aux[1] / tile_size;
ArrayList<ArrayList<DS>> ds_list = new ArrayList<ArrayList<DS>>();
for (int nt = 0; nt < tilesX_aux * tilesY_aux; nt++) {
ds_list.add(new ArrayList<DS>());
}
for (int ty = 0; ty < tilesY_main; ty++) {
double centerY = ty * tile_size + tile_size/2;
for (int tx = 0; tx < tilesX_main; tx++) {
int nt = ty*tilesX_main + tx;
double centerX = tx * tile_size + tile_size/2;
double disparity = ds[0][nt];
double strength = ds[1][nt];
if ((strength >= min_strength) && !Double.isNaN(disparity)) {
double [] dpxpy_aux = geometryCorrection_aux.getFromOther(
geometryCorrection_main, // GeometryCorrection other_gc,
centerX, // double other_px,
centerY, // double other_py,
disparity); // double other_disparity)
double fx = dpxpy_aux[1]/tile_size;
double fy = dpxpy_aux[2]/tile_size;
int tx_aux = (int) Math.floor(fx);
int ty_aux = (int) Math.floor(fy);
fx -= tx_aux;
fy -= ty_aux;
if ((ty_aux >= 0) && (ty_aux < tilesY_aux) && (tx_aux >= 0) && (tx_aux < tilesX_aux)) {
int nt_aux = ty_aux * tilesX_aux + tx_aux;
ds_list.get(nt_aux).add(new DS(dpxpy_aux[0], strength, tx, ty, fx, fy));
}
}
}
}
// simple average (ignoring below minimal)
int num_slices = split_fg_bg? 8:2;
double [][] ds_aux_avg = new double [num_slices][tilesX_aux * tilesY_aux];
for (int ty = 0; ty < tilesY_aux; ty++) {
for (int tx = 0; tx < tilesX_aux; tx++) {
if ((ty == 4) && (tx == 12)) {
System.out.println("tx = "+tx+", ty = "+ty);
}
int nt = ty * tilesX_aux + tx;
ds_aux_avg[0][nt] = Double.NaN;
ds_aux_avg[1][nt] = 0.0;
if(ds_list.get(nt).isEmpty()) continue;
Collections.sort(ds_list.get(nt), new Comparator<DS>() {
@Override
public int compare(DS lhs, DS rhs) { // ascending
return rhs.disparity > lhs.disparity ? -1 : (rhs.disparity < lhs.disparity ) ? 1 : 0;
}
});
double sw = 0.0, swd = 0.0, swd2 = 0.0;
for (DS dsi: ds_list.get(nt)) {
double w = dsi.strength;
if (use_wnd) {
w *= Math.sin(Math.PI * (dsi.fx + 0.5/tile_size)) * Math.sin(Math.PI * (dsi.fy + 0.5/tile_size));
}
sw += w;
double wd = w * dsi.disparity;
swd += wd;
swd2 += wd * dsi.disparity;
}
ds_aux_avg[0][nt] = swd/sw;
ds_aux_avg[1][nt] = sw/ds_list.get(nt).size();
if (split_fg_bg) {
ds_aux_avg[2][nt] = Math.sqrt( (swd2 * sw - swd * swd) / (sw * sw));
ds_aux_avg[3][nt] = ds_aux_avg[2][nt]; // rms
ds_aux_avg[4][nt] = ds_aux_avg[0][nt]; // fg disp
ds_aux_avg[5][nt] = ds_aux_avg[1][nt]; // fg strength
ds_aux_avg[6][nt] = ds_aux_avg[0][nt]; // bg disp
ds_aux_avg[7][nt] = ds_aux_avg[1][nt]; // bg strength
if (ds_aux_avg[2][nt] >= split_fbg_rms) {
// splitting while minimizing sum of 2 squared errors
double [][] swfb = new double [2][ds_list.get(nt).size() -1];
double [][] swdfb = new double [2][ds_list.get(nt).size() -1];
double [] s2fb = new double [ds_list.get(nt).size() -1];
for (int n = 0; n < s2fb.length; n++) { // split position
double [] s2 = new double[2];
for (int i = 0; i <= s2fb.length; i++) {
int fg = (i > n)? 1 : 0; // 0 - bg, 1 - fg
DS dsi = ds_list.get(nt).get(i);
double w = dsi.strength;
if (use_wnd) {
w *= Math.sin(Math.PI * dsi.fx) * Math.sin(Math.PI * dsi.fy);
}
swfb[fg][n] += w;
double wd = w * dsi.disparity;
swdfb[fg][n] += wd;
s2[fg] += wd * dsi.disparity;
}
s2fb[n] = ((s2[0] * swfb[0][n] - swdfb[0][n] * swdfb[0][n]) / swfb[0][n] +
(s2[1] * swfb[1][n] - swdfb[1][n] * swdfb[1][n]) / swfb[1][n]) / (swfb[0][n] + swfb[1][n]);
}
// now find the n with lowest s2fb and use it to split fg/bg. Could be done in a single pass, but with saved arrays
// it is easier to verify
int nsplit = 0;
for (int i = 1; i < s2fb.length; i++) if (s2fb[i] < s2fb[nsplit]) {
nsplit = i;
}
ds_aux_avg[3][nt] = s2fb[nsplit]; // rms split
ds_aux_avg[4][nt] = swdfb[1][nsplit] / swfb[1][nsplit] ; // fg disp
ds_aux_avg[5][nt] = swfb[1][nsplit]/ (s2fb.length - nsplit) ; // fg strength
ds_aux_avg[6][nt] = swdfb[0][nsplit] / swfb[0][nsplit] ; // bg disp
ds_aux_avg[7][nt] = swfb[0][nsplit]/ (nsplit + 1) ; // bg strength
}
}
}
}
return ds_aux_avg;
}
//getGeometryCorrection
///data_ssd/lwir3d/results/saved/1562390490_233403/v11/
/* ======================================================================== */
......
src/main/java/com/elphel/imagej/tileprocessor/AlignmentCorrection.java
View file @ be9da419
......@@ -2311,7 +2311,7 @@ B = |+dy0 -dy1 -2*dy3 |
clt_parameters.ly_par_sel, //int manual_par_sel, // Manually select the parameter mask bit 0 - sym0, bit1 - sym1, ... (0 - use boolean flags, != 0 - ignore boolean flags)
mismatch_list, // ArrayList<Mismatch> mismatch_list,
qc.geometryCorrection, // GeometryCorrection geometryCorrection,
null, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels
/// null, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels
qc.geometryCorrection.getCorrVector(), // GeometryCorrection.CorrVector corr_vector,
old_new_rms, // double [] old_new_rms, // should be double[2]
// 2); // debugLevel); // 2); // 1); // int debugLevel)
......@@ -2360,7 +2360,7 @@ B = |+dy0 -dy1 -2*dy3 |
public double [][][] lazyEyeCorrectionFromGT(
final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig
// final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig
final boolean use_poly, // Use polynomial correction, false - correct tilt/azimuth/roll of each sensor
final boolean restore_disp_inf, // Restore subtracted disparity for scan #0 (infinity) always true
final double fcorr_radius,
......@@ -2398,7 +2398,6 @@ B = |+dy0 -dy1 -2*dy3 |
int debugLevel){
// final double lazyEyeDispRelVariation = 0.02;
//geometryCorrection_main
final int dbg_nTile = -34145; // 37005; // -59038;
final int num_scans = scans_14.length/NUM_ALL_SLICES;
final int num_tiles = scans_14[0].length;
......@@ -2791,7 +2790,7 @@ B = |+dy0 -dy1 -2*dy3 |
clt_parameters.ly_par_sel, // int manual_par_sel, // Manually select the parameter mask bit 0 - sym0, bit1 - sym1, ... (0 - use boolean flags, != 0 - ignore boolean flags)
mismatch_list, // ArrayList<Mismatch> mismatch_list,
qc.geometryCorrection, // GeometryCorrection geometryCorrection,
geometryCorrection_main, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels
/// geometryCorrection_main, // GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels
qc.geometryCorrection.getCorrVector(), // GeometryCorrection.CorrVector corr_vector,
old_new_rms, // double [] old_new_rms, // should be double[2]
// 2); // debugLevel); // 2); // 1); // int debugLevel)
......@@ -3326,7 +3325,7 @@ B = |+dy0 -dy1 -2*dy3 |
int manual_par_sel, // Manually select the parameter mask bit 0 - sym0, bit1 - sym1, ... (0 - use boolean flags, != 0 - ignore boolean flags)
ArrayList<Mismatch> mismatch_list,
GeometryCorrection geometryCorrection,
GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels
/// GeometryCorrection geometryCorrection_main, // if is aux camera using main cameras' coordinates. Disparity is still in aux camera pixels
GeometryCorrection.CorrVector corr_vector,
double [] old_new_rms, // should be double[2]
int debugLevel)
......
src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java
View file @ be9da419
......@@ -34,6 +34,8 @@ import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Properties;
import java.util.Set;
......@@ -101,7 +103,8 @@ public class QuadCLT {
// is enabled
double [] lwir_cold_hot = null;
// int [] woi_tops; // used to calculate scanline timing
// just for debugging with the use of intermediate image
public double [][] ds_from_main = null;
// magic scale should be set before using TileProcessor (calculated disparities depend on it)
public boolean isMonochrome() {return is_mono;}
......@@ -5999,21 +6002,36 @@ public class QuadCLT {
tp.ShowScansSFB(
combo_pass_list, // ArrayList<CLTPass3d> scans, // list of composite scans
this.image_name+"-SFB"); // String title);
/*
if (show_init_refine) tp.showScan(
combo_pass, // CLTPass3d scan,
"after_compositeScan-"+tp.clt_3d_passes.size());
*/
}
if (adjust_extrinsics) {
if (use_rig) {
// temporarily
if (ds_from_main != null) {
System.out.println("Adjust AUX extrinsics using main camera measurements");
extrinsicsCLTfromGT(
// twoQuadCLT, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
null,
ds_from_main, // gt_disp_strength,
clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters,
adjust_poly,
threadsMax, //final int threadsMax, // maximal number of threads to launch
updateStatus,// final boolean updateStatus,
debugLevel + 2); // final int debugLevel)
} else if (use_rig) {
System.out.println("Adjust extrinsics using rig data here");
double [][] gt_disp_strength = getRigDSFromTwoQuadCL(
twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
clt_parameters,
debugLevel + 2); // final int debugLevel)
GeometryCorrection geometryCorrection_main = null;
if (geometryCorrection.getRotMatrix(true) != null) {
geometryCorrection_main = twoQuadCLT.quadCLT_main.getGeometryCorrection();
}
extrinsicsCLTfromGT(
twoQuadCLT, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
// twoQuadCLT, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
geometryCorrection_main,
gt_disp_strength,
clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters,
adjust_poly,
threadsMax, //final int threadsMax, // maximal number of threads to launch
......@@ -6084,6 +6102,167 @@ public class QuadCLT {
IJ.d2s(0.000000001*(System.nanoTime()-this.startTime),3)+" sec, --- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")");
}
public double [][] depthMapMainToAux(
double [][] ds,
GeometryCorrection geometryCorrection_main,
GeometryCorrection geometryCorrection_aux,
CLTParameters clt_Parameters,
// double min_strength,
// boolean use_wnd,
boolean split_fg_bg,
// double split_fbg_rms,
boolean for_adjust, // for LY adjustment: only keep d,s and remove samples with high variations
int debug_level
){
class DS{
double disparity; // gt disparity
double strength; // gt strength
int tx; // gt tile x
int ty; // gt tile x
double fx; // fractional aux tile X (0.0..1.0) for optional window
double fy; // fractional aux tile Y (0.0..1.0) for optional window
DS (double disparity, double strength, int tx, int ty, double fx, double fy){
this.disparity = disparity;
this.strength = strength;
this.tx = tx;
this.ty = ty;
this.fx = fx;
this.fy = fy;
}
@Override
public String toString() {
return String.format("Disparity (str) = % 6f (%5f), tx=%d ty=%d fx=%5f fy=%5f\n", disparity, strength,tx,ty,fx,fy);
}
}
int tile_size = clt_Parameters.transform_size;
int [] wh_main = geometryCorrection_main.getSensorWH();
int [] wh_aux = geometryCorrection_aux.getSensorWH();
int tilesX_main = wh_main[0] / tile_size;
int tilesY_main = wh_main[1] / tile_size;
int tilesX_aux = wh_aux[0] / tile_size;
int tilesY_aux = wh_aux[1] / tile_size;
ArrayList<ArrayList<DS>> ds_list = new ArrayList<ArrayList<DS>>();
for (int nt = 0; nt < tilesX_aux * tilesY_aux; nt++) {
ds_list.add(new ArrayList<DS>());
}
for (int ty = 0; ty < tilesY_main; ty++) {
double centerY = ty * tile_size + tile_size/2;
for (int tx = 0; tx < tilesX_main; tx++) {
int nt = ty*tilesX_main + tx;
double centerX = tx * tile_size + tile_size/2;
double disparity = ds[0][nt];
double strength = ds[1][nt];
if ((strength >= clt_Parameters.ly_gt_strength) && !Double.isNaN(disparity)) {
double [] dpxpy_aux = geometryCorrection_aux.getFromOther(
geometryCorrection_main, // GeometryCorrection other_gc,
centerX, // double other_px,
centerY, // double other_py,
disparity); // double other_disparity)
double fx = dpxpy_aux[1]/tile_size;
double fy = dpxpy_aux[2]/tile_size;
int tx_aux = (int) Math.floor(fx);
int ty_aux = (int) Math.floor(fy);
fx -= tx_aux;
fy -= ty_aux;
if ((ty_aux >= 0) && (ty_aux < tilesY_aux) && (tx_aux >= 0) && (tx_aux < tilesX_aux)) {
int nt_aux = ty_aux * tilesX_aux + tx_aux;
ds_list.get(nt_aux).add(new DS(dpxpy_aux[0], strength, tx, ty, fx, fy));
}
}
}
}
// simple average (ignoring below minimal)
int num_slices = split_fg_bg? 8:2;
double [][] ds_aux_avg = new double [num_slices][tilesX_aux * tilesY_aux];
for (int ty = 0; ty < tilesY_aux; ty++) {
for (int tx = 0; tx < tilesX_aux; tx++) {
if ((ty == 3) && (tx == 12)) {
System.out.println("tx = "+tx+", ty = "+ty);
}
int nt = ty * tilesX_aux + tx;
ds_aux_avg[0][nt] = Double.NaN;
ds_aux_avg[1][nt] = 0.0;
if(ds_list.get(nt).isEmpty()) continue;
Collections.sort(ds_list.get(nt), new Comparator<DS>() {
@Override
public int compare(DS lhs, DS rhs) { // ascending
return rhs.disparity > lhs.disparity ? -1 : (rhs.disparity < lhs.disparity ) ? 1 : 0;
}
});
double sw = 0.0, swd = 0.0, swd2 = 0.0;
for (DS dsi: ds_list.get(nt)) {
double w = dsi.strength;
if (clt_Parameters.ly_gt_use_wnd) {
w *= Math.sin(Math.PI * dsi.fx) * Math.sin(Math.PI * dsi.fy);
}
sw += w;
double wd = w * dsi.disparity;
swd += wd;
swd2 += wd * dsi.disparity;
}
ds_aux_avg[0][nt] = swd/sw;
ds_aux_avg[1][nt] = sw/ds_list.get(nt).size();
double rms = Math.sqrt( (swd2 * sw - swd * swd) / (sw * sw));
if (for_adjust && (rms >= clt_Parameters.ly_gt_rms)) { // remove ambiguous tiles
ds_aux_avg[0][nt] = Double.NaN;
ds_aux_avg[1][nt] = 0;
}
if (split_fg_bg) {
ds_aux_avg[2][nt] = rms;
ds_aux_avg[3][nt] = ds_aux_avg[2][nt]; // rms
ds_aux_avg[4][nt] = ds_aux_avg[0][nt]; // fg disp
ds_aux_avg[5][nt] = ds_aux_avg[1][nt]; // fg strength
ds_aux_avg[6][nt] = ds_aux_avg[0][nt]; // bg disp
ds_aux_avg[7][nt] = ds_aux_avg[1][nt]; // bg strength
if (rms >= clt_Parameters.ly_gt_rms) {
// splitting while minimizing sum of 2 squared errors
double [][] swfb = new double [2][ds_list.get(nt).size() -1];
double [][] swdfb = new double [2][ds_list.get(nt).size() -1];
double [] s2fb = new double [ds_list.get(nt).size() -1];
for (int n = 0; n < s2fb.length; n++) { // split position
double [] s2 = new double[2];
for (int i = 0; i <= s2fb.length; i++) {
int fg = (i > n)? 1 : 0; // 0 - bg, 1 - fg
DS dsi = ds_list.get(nt).get(i);
double w = dsi.strength;
if (clt_Parameters.ly_gt_use_wnd) {
w *= Math.sin(Math.PI * dsi.fx) * Math.sin(Math.PI * dsi.fy);
}
swfb[fg][n] += w;
double wd = w * dsi.disparity;
swdfb[fg][n] += wd;
s2[fg] += wd * dsi.disparity;
}
s2fb[n] = ((s2[0] * swfb[0][n] - swdfb[0][n] * swdfb[0][n]) / swfb[0][n] +
(s2[1] * swfb[1][n] - swdfb[1][n] * swdfb[1][n]) / swfb[1][n]) / (swfb[0][n] + swfb[1][n]);
}
// now find the n with lowest s2fb and use it to split fg/bg. Could be done in a single pass, but with saved arrays
// it is easier to verify
int nsplit = 0;
for (int i = 1; i < s2fb.length; i++) if (s2fb[i] < s2fb[nsplit]) {
nsplit = i;
}
ds_aux_avg[3][nt] = s2fb[nsplit]; // rms split
ds_aux_avg[4][nt] = swdfb[1][nsplit] / swfb[1][nsplit] ; // fg disp
ds_aux_avg[5][nt] = swfb[1][nsplit]/ (s2fb.length - nsplit) ; // fg strength
ds_aux_avg[6][nt] = swdfb[0][nsplit] / swfb[0][nsplit] ; // bg disp
ds_aux_avg[7][nt] = swfb[0][nsplit]/ (nsplit + 1) ; // bg strength
}
}
}
}
return ds_aux_avg;
}
public boolean preExpandCLTQuad3d(
ImagePlus [] imp_quad, // should have properties "name"(base for saving results), "channel","path"
boolean [][] saturation_imp, // (near) saturated pixels or null
......@@ -6903,9 +7082,42 @@ public class QuadCLT {
}
public boolean extrinsicsCLTfromGT(
public double [][] getRigDSFromTwoQuadCL(
TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
CLTParameters clt_parameters,
final int debugLevel) {
if ((twoQuadCLT == null) || (twoQuadCLT.getBiScan(0) == null)){
System.out.println("Rig data is not available, aborting");
return null;
}
BiScan scan = twoQuadCLT.getBiScan(0);
double [][] rig_disp_strength = scan.getDisparityStrength(
true, // final boolean only_strong,
true, // final boolean only_trusted,
true) ; // final boolean only_enabled);
GeometryCorrection geometryCorrection_main = null;
if (geometryCorrection.getRotMatrix(true) != null) {
geometryCorrection_main = twoQuadCLT.quadCLT_main.getGeometryCorrection();
double disparityScale = geometryCorrection.getDisparityRadius()/geometryCorrection_main.getDisparityRadius();
for (int i = 0; i < rig_disp_strength[0].length; i++) {
rig_disp_strength[0][i] *= disparityScale;
}
if (debugLevel > -2) {
System.out.println("This is an AUX camera, using MAIN camera coordinates");
}
}
return rig_disp_strength;
}
public boolean extrinsicsCLTfromGT(
// TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
GeometryCorrection geometryCorrection_main, // only used for aux camera if coordinates are for main (null for LWIR)
double [][] rig_disp_strength,
CLTParameters clt_parameters,
boolean adjust_poly,
final int threadsMax, // maximal number of threads to launch
final boolean updateStatus,
......@@ -6920,6 +7132,7 @@ public class QuadCLT {
int max_tries = clt_parameters.lym_iter; // 25;
double min_sym_update = clt_parameters.getLymChange(is_aux); // 4e-6; // stop iterations if no angle changes more than this
double min_poly_update = clt_parameters.lym_poly_change; // Parameter vector difference to exit from polynomial correction
/*
if ((twoQuadCLT == null) || (twoQuadCLT.getBiScan(0) == null)){
System.out.println("Rig data is not available, aborting");
return false;
......@@ -6951,8 +7164,17 @@ public class QuadCLT {
if (debugLevel > -2) {
System.out.println("This is an AUX camera, using MAIN camera coordinates");
}
}
*/
if (debugLevel > 20) {
boolean tmp_exit = true;
System.out.println("extrinsicsCLTfromGT()");
if (tmp_exit) {
System.out.println("will now exit. To continue - change variable tmp_exit in debugger" );
if (tmp_exit) {
return false;
}
}
}
CLTPass3d comboScan = tp.compositeScan(
......@@ -6966,7 +7188,6 @@ public class QuadCLT {
// iteration steps
double comp_diff = min_sym_update + 1; // (> min_sym_update)
for (int num_iter = 0; num_iter < max_tries; num_iter++){
double [][] combo_mismatch = new double[12][];
CLTMeasure( // perform single pass according to prepared tiles operations and disparity
image_data, // first index - number of image in a quad
......@@ -6998,10 +7219,9 @@ public class QuadCLT {
}
double [][][] new_corr;
// final double inf_max_disparity = 2.0;
double [][][] gt_disparity_strength = {rig_disp_strength};
new_corr = ac.lazyEyeCorrectionFromGT(
geometryCorrection_main, //final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig
/// geometryCorrection_main, //final GeometryCorrection geometryCorrection_main, // if not null - this is an AUX camera of a rig
adjust_poly, // final boolean use_poly,
true, // final boolean restore_disp_inf, // Restore subtracted disparity for scan #0 (infinity)
clt_parameters.fcorr_radius, // final double fcorr_radius,
......
src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java
View file @ be9da419
......@@ -3680,7 +3680,7 @@ if (debugLevel > -100) return true; // temporarily !
// Get DSI from the main camera
quadCLT_main.tp.trimCLTPasses(false); // remove rig composite scan if any
// last but not including any rid data
// last but not including any rig data
CLTPass3d scan_last = quadCLT_main.tp.clt_3d_passes.get( quadCLT_main.tp.clt_3d_passes_size -1); // get last one
double [][] disparity_bimap = setBimapFromCLTPass3d(
scan_last, // CLTPass3d scan,
......@@ -7445,14 +7445,25 @@ if (debugLevel > -100) return true; // temporarily !
System.out.println("Adjusting main camera image set for "+quadCLT_main.image_name+
" (with rig DSI), pass "+(num_adjust_main+1)+" of "+quadCLT_main.correctionsParameters.rig_batch_adjust_main_gt);
}
double [][] gt_disp_strength = quadCLT_main.getRigDSFromTwoQuadCL(
this, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
clt_parameters,
debugLevelInner); // final int debugLevel)
// GeometryCorrection geometryCorrection_main = null;
// if (geometryCorrection.getRotMatrix(true) != null) {
// geometryCorrection_main = twoQuadCLT.quadCLT_main.getGeometryCorrection();
// }
quadCLT_main.extrinsicsCLTfromGT(
this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters,
false,
threadsMax, //final int threadsMax, // maximal number of threads to launch
updateStatus,// final boolean updateStatus,
debugLevelInner); // final int debugLevel)
// this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
null,
gt_disp_strength,
clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters,
false,
threadsMax, //final int threadsMax, // maximal number of threads to launch
updateStatus,// final boolean updateStatus,
debugLevelInner); // final int debugLevel)
}
for (int num_adjust_aux = 0; num_adjust_aux < quadCLT_main.correctionsParameters.rig_batch_adjust_aux_gt; num_adjust_aux++) {
......@@ -7462,9 +7473,15 @@ if (debugLevel > -100) return true; // temporarily !
System.out.println("Adjusting aux camera image set for "+quadCLT_main.image_name+
" (with rig DSI), pass "+(num_adjust_aux+1)+" of "+quadCLT_main.correctionsParameters.rig_batch_adjust_aux_gt);
}
double [][] gt_disp_strength = quadCLT_aux.getRigDSFromTwoQuadCL(
this, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
clt_parameters,
debugLevelInner); // final int debugLevel)
quadCLT_aux.extrinsicsCLTfromGT(
this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
// this, // TwoQuadCLT twoQuadCLT, //maybe null in no-rig mode, otherwise may contain rig measurements to be used as infinity ground truth
quadCLT_main.getGeometryCorrection(),
gt_disp_strength,
clt_parameters, // EyesisCorrectionParameters.CLTParameters clt_parameters,
false,
threadsMax, //final int threadsMax, // maximal number of threads to launch
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment