Skip to content

I
imagej-elphel
Commit defa789f authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

Better pixel offset estimation

parent 1f6a9eb9
Show whitespace changes
Inline Side-by-side
Showing with 507 additions and 114 deletions
src/main/java/com/elphel/imagej/tileprocessor/CorrVector.java
View file @ defa789f
......@@ -626,10 +626,22 @@ public class CorrVector{ // TODO: Update to non-quad (extract to a file first)?
return toString(false);
}
/**
* Astimate azimuth/tilt angle for 1 pixel shift
* @return Angle in radians to rotate image by 1 pixel by 1 pixel
*/
public double getTiltAzPerPixel() {
return 1.0/1000.0*geometryCorrection.focalLength/geometryCorrection.pixelSize;
}
/**
* Estimate roll angle for average pixel shift of 1 pixel
* @return Angle in radians when pixels at 1/4 sensor width rotate by 1 pixel
*/
public double getRollPixel() {
return 4.0/geometryCorrection.getSensorWH()[0];
}
public String toString(boolean short_out) {
String s;
double [] sym_vect = toSymArray(null);
......
src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java
View file @ defa789f
......@@ -1280,7 +1280,7 @@ public class ImageDtt extends ImageDttCPU {
gpuQuad.execConvertDirect(use_reference_buffer, wh, erase_clt); // put results into a "reference" buffer
}
public void setReferenceTDMotionBlur(
public void setReferenceTDMotionBlur( // updates tasks
final int erase_clt,
final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
......
src/main/java/com/elphel/imagej/tileprocessor/Interscene.java
View file @ defa789f
......@@ -371,6 +371,7 @@ public class Interscene {
double [] lma_rms = new double[2];
int tilesX = quadCLTs[ref_index].getTileProcessor().getTilesX();
int tilesY = quadCLTs[ref_index].getTileProcessor().getTilesY();
int tile_size = quadCLTs[ref_index].getTileProcessor().getTileSize();
double min_offset = 0.0; // clt_parameters.imp.min_offset;
double max_offset = clt_parameters.imp.max_rel_offset * tilesX * tile_size;
......@@ -458,6 +459,13 @@ public class Interscene {
QuadCLTCPU.scaleDtToErs(
clt_parameters,
cam_dxyzatr_ref));
// Trying new version with motion blur and single-setting of the reference frame
boolean mb_en = clt_parameters.imp.mb_en;
double mb_tau = clt_parameters.imp.mb_tau; // 0.008; // time constant, sec
double mb_max_gain = clt_parameters.imp.mb_max_gain_inter; // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
TpTask[][] tp_tasks_ref = new TpTask[2][];
double [][] pXpYD_ref = new double [tilesX*tilesY][];
for (int scene_index = ref_index - 1; scene_index >= earliest_scene ; scene_index--) {
if ((ref_index - scene_index) >= max_num_scenes){
earliest_scene = scene_index + 1;
......@@ -534,7 +542,7 @@ public class Interscene {
QuadCLTCPU.scaleDtToErs(
clt_parameters,
cam_dxyzatr));
/*
// Does not use motion blur for reference scene here!
scenes_xyzatr[scene_index] = adjustPairsLMAInterscene(
clt_parameters, // CLTParameters clt_parameters,
......@@ -556,6 +564,35 @@ public class Interscene {
lma_rms, // double [] rms, // null or double [2]
clt_parameters.imp.max_rms, // double max_rms,
clt_parameters.imp.debug_level); // 1); // -1); // int debug_level);
*/
// Trying new version with motion blur and single-setting of the reference frame
scenes_xyzatr[scene_index] = adjustDiffPairsLMAInterscene( // compare two scenes, first may be reference, use motion blur
clt_parameters, // CLTParameters clt_parameters,
use_lma_dsi, // clt_parameters.imp.use_lma_dsi,
false, // boolean fpn_disable, // disable fpn filter if images are known to be too close
true, // boolean disable_ers,
min_max, // double [] min_max, // null or pair of minimal and maximal offsets
fail_reason, // int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max
quadCLTs, // QuadCLT [] quadCLTs,
ref_index, // int ref_index,
tp_tasks_ref, // TpTask[][] tp_tasks_ref, // Should be TpTask[2][*]. If tp_tasks_ref[0] == null will calculate
// at set first scene to the GPU
pXpYD_ref, // double [][] pXpYD_ref, // should be se or at least double [num_tiles][] if tp_tasks_ref[0] == null
ref_index, // int nscene0, // may be == ref_index
scene_index, // int nscene1, // compares to nscene0
null, // double [] ref_disparity, // null or alternative reference disparity
reliable_ref, //boolean [] reliable_ref, // null or bitmask of reliable reference tiles
scenes_xyzatr[ref_index], // double [][] scene0_xyzatr,,
initial_pose, // double [][] scene1_xyzatr,
initial_pose, // double [] scene1_xyzatr_pull, // if both are not null, specify target values to pull to
clt_parameters.ilp.ilma_lma_select, // boolean[] param_select,
reg_weights, // double [] param_regweights,
lma_rms, // double [] rms_out, // null or double [2]
clt_parameters.imp.max_rms, // double max_rms,
mb_en, // boolean mb_en,
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
clt_parameters.imp.debug_level); // int debugLevel);
boolean adjust_OK = scenes_xyzatr[scene_index] != null;
if (adjust_OK) { // check only for initial orientation, do not check on readjustments
......@@ -1061,6 +1098,102 @@ public class Interscene {
return earliest_scene;
}
public static void readjustShowCorrected(
CLTParameters clt_parameters,
QuadCLT [] quadCLTs,
double [][] dxyzatr_dt_nscene,
double [][] dbg_mb_img, // initialize to double[quadCLTs.length][]
double mb_max_gain,
double [] mb_ref_disparity,
int ref_index,
int nscene,
int debug_scene, // = 68
int debugLevel) {
boolean show_corrected = false;
double mb_tau = clt_parameters.imp.mb_tau; // 0.008; // time constant, sec
int tilesX = quadCLTs[ref_index].getTileProcessor().getTilesX();
int tilesY = quadCLTs[ref_index].getTileProcessor().getTilesY();
ErsCorrection ers_reference = quadCLTs[ref_index].getErsCorrection();
if (nscene == debug_scene) {
System.out.println("2.nscene = "+nscene);
System.out.println("3.nscene = "+nscene);
}
dbg_mb_img[nscene] = new double [tilesX*tilesY*2];
Arrays.fill(dbg_mb_img[nscene],Double.NaN);
String ts = quadCLTs[nscene].getImageName();
double [] mb_scene_xyz = (nscene != ref_index)? ers_reference.getSceneXYZ(ts):ZERO3;
double [] mb_scene_atr = (nscene != ref_index)? ers_reference.getSceneATR(ts):ZERO3;
double [][] ref_pXpYD = OpticalFlow.transformToScenePxPyD( // full size - [tilesX*tilesY], some nulls
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
mb_ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
quadCLTs[ref_index]); // final QuadCLT reference_QuadClt)
double [][] motion_blur = OpticalFlow.getMotionBlur( // dxyzatr_dt[][] should not be scaled here
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[nscene], // QuadCLT scene, // can be the same as ref_scene
ref_pXpYD, // double [][] ref_pXpYD, // here it is scene, not reference!
mb_scene_xyz, // double [] camera_xyz,
mb_scene_atr, // double [] camera_atr,
dxyzatr_dt_nscene[0], // double [] camera_xyz_dt,
dxyzatr_dt_nscene[1], // double [] camera_atr_dt,
-1, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
for (int nTile = 0; nTile < motion_blur[0].length; nTile++) {
int tx = nTile % tilesX;
int ty = nTile / tilesX;
dbg_mb_img[nscene][tx + tilesX * (ty*2 +0)] = mb_tau * motion_blur[0][nTile];
dbg_mb_img[nscene][tx + tilesX * (ty*2 +1)] = mb_tau * motion_blur[1][nTile];
}
while (show_corrected) {
ImagePlus imp_mbc = QuadCLT.renderGPUFromDSI(
-1, // final int sensor_mask,
false, // final boolean merge_channels,
null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
clt_parameters, // CLTParameters clt_parameters,
mb_ref_disparity, // double [] disparity_ref,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
motion_blur, // double [][] mb_vectors, //
mb_scene_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
mb_scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene], // final QuadCLT scene,
quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
false, // toRGB, // final boolean toRGB,
clt_parameters.imp.show_color_nan,
quadCLTs[nscene].getImageName()+"-MOTION_BLUR_CORRECTED", // String suffix,
THREADS_MAX, // int threadsMax,
debugLevel); // int debugLevel)
imp_mbc.show();
ImagePlus imp_mbc_merged = QuadCLT.renderGPUFromDSI(
-1, // final int sensor_mask,
true, // final boolean merge_channels,
null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
clt_parameters, // CLTParameters clt_parameters,
mb_ref_disparity, // double [] disparity_ref,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
motion_blur, // double [][] mb_vectors, //
mb_scene_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
mb_scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene], // final QuadCLT scene,
quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
false, // toRGB, // final boolean toRGB,
clt_parameters.imp.show_color_nan,
quadCLTs[nscene].getImageName()+"-MOTION_BLUR_CORRECTED-MERGED", // String suffix,
THREADS_MAX, // int threadsMax,
debugLevel); // int debugLevel)
imp_mbc_merged.show();
show_corrected = false; // manually set/reset
}
}
public static int reAdjustPairsLMAInterscene( // after combo dgi is available and preliminary poses are known
CLTParameters clt_parameters,
......@@ -1082,6 +1215,7 @@ public class Interscene {
double half_run_range = clt_parameters.ilp.ilma_motion_filter; // 3.50; // make a parameter
double [][] dbg_scale_dt = {clt_parameters.ilp.ilma_scale_xyz, clt_parameters.ilp.ilma_scale_atr};
boolean debug_ers = clt_parameters.ilp.ilma_debug_ers ; // true;
double avg_z = quadCLTs[ref_index].getAverageZ(true); // for offset estimation
String dbg_ers_string =
((dbg_scale_dt[1][0] > 0)?"a":((dbg_scale_dt[1][0] < 0) ? "A":"0"))+
......@@ -1183,7 +1317,7 @@ public class Interscene {
scenes_xyzatr[ref_index] = new double[2][3]; // all zeros
// should have at least next or previous non-null
int debug_scene = 68; // -8;
int debug_scene = 69; // -68; // -8;
double maximal_series_rms = 0.0;
double [][] mb_vectors_ref = null;
TpTask[][] tp_tasks_ref = null;
......@@ -1211,10 +1345,40 @@ public class Interscene {
}
dxyzatr_dt[nscene] = ers_reference.getSceneErsXYZATR_dt(quadCLTs[nscene].getImageName());
if (debug_ers) {
System.out.println(String.format("%3d xyz: %9.5f %9.5f %9.5f atr: %9.5f %9.5f %9.5f",
// double est_pix = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true);
// double est_pix1 = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true,1);
// double est_pix2 = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true,2);
// double est_pix3 = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true,3);
double est_center = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
false, // boolean use_rot,
false); // boolean rectilinear)
double est_quad = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
true, // boolean use_rot,
false); // boolean rectilinear)
double est_center_rectilinear = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
false, // boolean use_rot,
true); // boolean rectilinear)
double est_quad_rectilinear = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
true, // boolean use_rot,
true); // boolean rectilinear)
System.out.println(String.format("%3d xyz: %9.5f %9.5f %9.5f atr: %9.5f %9.5f %9.5f est_pix: %9.4f %9.4f %9.4f %9.4f",
nscene,
scenes_xyzatr[nscene][0][0], scenes_xyzatr[nscene][0][1], scenes_xyzatr[nscene][0][2],
scenes_xyzatr[nscene][1][0], scenes_xyzatr[nscene][1][1], scenes_xyzatr[nscene][1][2]));
scenes_xyzatr[nscene][1][0], scenes_xyzatr[nscene][1][1], scenes_xyzatr[nscene][1][2],
// est_pix, est_pix1, est_pix2, est_pix3,
est_center, est_quad, est_center_rectilinear, est_quad_rectilinear));
}
}
switch (ers_mode) {
......@@ -1248,7 +1412,7 @@ public class Interscene {
}
double max_z_change = Double.NaN; // only applicable for drone images
if (max_zoom_diff > 0) { // ignore if set to
double avg_z = quadCLTs[ref_index].getAverageZ(true); // use lma
// double avg_z = quadCLTs[ref_index].getAverageZ(true); // use lma
max_z_change = avg_z * max_zoom_diff;
if (fail_on_zoom_roll) {
if (debugLevel > -3) {
......@@ -1273,6 +1437,40 @@ public class Interscene {
// for (int nscene = ref_index; nscene >= earliest_scene; nscene--) {
for (int nscene:scene_seq) {
if (debugLevel > -4) {
// double est_pix = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true);
// double est_pix1 = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true,1);
// double est_pix2 = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true,2);
// double est_pix3 = quadCLTs[ref_index].estimatePixelShift(scenes_xyzatr[ref_index], scenes_xyzatr[nscene], true,3);
double est_center = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
false, // boolean use_rot,
false); // boolean rectilinear)
double est_quad = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
true, // boolean use_rot,
false); // boolean rectilinear)
double est_center_rectilinear = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
false, // boolean use_rot,
true); // boolean rectilinear)
double est_quad_rectilinear = quadCLTs[ref_index].estimateAverageShift(
scenes_xyzatr[ref_index], // double [][] xyzatr0,
scenes_xyzatr[nscene], // double [][] xyzatr1,
avg_z, // double average_z,
true, // boolean use_rot,
true); // boolean rectilinear)
// only check quad if center is below thershold (not to get probed too far - overlap)
// rectilinear - OK, error is small.
System.out.println("nscene="+nscene+": est offset "+
est_center+"pix "+est_quad+"pix "+est_center_rectilinear+"pix "+est_quad_rectilinear+"pix ");
}
if (nscene == debug_scene) {
System.out.println("nscene = "+nscene);
System.out.println("nscene = "+nscene);
......@@ -1290,103 +1488,26 @@ public class Interscene {
double [] scene_xyz_pre = ZERO3;
double [] scene_atr_pre = ZERO3;
// find correct signs when setting. dxyzatr_dt[][] is also used for motion blur (correctly)
/*
double [][] scaled_dxyzatr_dt = new double[2][3];
for (int i = 0; i < scaled_dxyzatr_dt.length; i++) {
for (int j = 0; j < scaled_dxyzatr_dt[i].length; j++) {
scaled_dxyzatr_dt[i][j] = dxyzatr_dt[nscene][i][j]* dbg_scale_dt[i][j];
}
}
// **** Here *_dt is set!
quadCLTs[nscene].getErsCorrection().setErsDt( // set for ref also (should be set before non-ref!)
scaled_dxyzatr_dt[0], // (ers_use_xyz? dxyzatr_dt[nscene][0]: ZERO3), //, // dxyzatr_dt[nscene][0], // double [] ers_xyz_dt,
scaled_dxyzatr_dt[1]); // dxyzatr_dt[nscene][1]); // double [] ers_atr_dt)(ers_scene_original_xyz_dt);
*/
quadCLTs[nscene].getErsCorrection().setErsDt( // set for ref also (should be set before non-ref!)
QuadCLTCPU.scaleDtToErs(
clt_parameters,
dxyzatr_dt[nscene]));
if (dbg_mb_img != null) {
boolean show_corrected = false;
if (nscene == debug_scene) {
System.out.println("2.nscene = "+nscene);
System.out.println("3.nscene = "+nscene);
}
dbg_mb_img[nscene] = new double [tilesX*tilesY*2];
Arrays.fill(dbg_mb_img[nscene],Double.NaN);
double [] mb_scene_xyz = (nscene != ref_index)? ers_reference.getSceneXYZ(ts):ZERO3;
double [] mb_scene_atr = (nscene != ref_index)? ers_reference.getSceneATR(ts):ZERO3;
double [][] motion_blur = OpticalFlow.getMotionBlur( // dxyzatr_dt[][] should not be scaled here
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[nscene], // QuadCLT scene, // can be the same as ref_scene
ref_pXpYD, // double [][] ref_pXpYD, // here it is scene, not reference!
mb_scene_xyz, // double [] camera_xyz,
mb_scene_atr, // double [] camera_atr,
dxyzatr_dt[nscene][0], // double [] camera_xyz_dt,
dxyzatr_dt[nscene][1], // double [] camera_atr_dt,
-1, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
for (int nTile = 0; nTile < motion_blur[0].length; nTile++) {
int tx = nTile % tilesX;
int ty = nTile / tilesX;
dbg_mb_img[nscene][tx + tilesX * (ty*2 +0)] = mb_tau * motion_blur[0][nTile];
dbg_mb_img[nscene][tx + tilesX * (ty*2 +1)] = mb_tau * motion_blur[1][nTile];
}
while (show_corrected) {
ImagePlus imp_mbc = QuadCLT.renderGPUFromDSI(
-1, // final int sensor_mask,
false, // final boolean merge_channels,
null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
if (dbg_mb_img != null) { // fill in 1 slice of dbg_mb_img, show all at once later
readjustShowCorrected(
clt_parameters, // CLTParameters clt_parameters,
mb_ref_disparity, // double [] disparity_ref,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
motion_blur, // double [][] mb_vectors, //
mb_scene_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
mb_scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene], // final QuadCLT scene,
quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
false, // toRGB, // final boolean toRGB,
clt_parameters.imp.show_color_nan,
quadCLTs[nscene].getImageName()+"-MOTION_BLUR_CORRECTED", // String suffix,
THREADS_MAX, // int threadsMax,
debugLevel); // int debugLevel)
imp_mbc.show();
ImagePlus imp_mbc_merged = QuadCLT.renderGPUFromDSI(
-1, // final int sensor_mask,
true, // final boolean merge_channels,
null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
clt_parameters, // CLTParameters clt_parameters,
mb_ref_disparity, // double [] disparity_ref,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
motion_blur, // double [][] mb_vectors, //
mb_scene_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
mb_scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene], // final QuadCLT scene,
quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
false, // toRGB, // final boolean toRGB,
clt_parameters.imp.show_color_nan,
quadCLTs[nscene].getImageName()+"-MOTION_BLUR_CORRECTED-MERGED", // String suffix,
THREADS_MAX, // int threadsMax,
debugLevel); // int debugLevel)
imp_mbc_merged.show();
}
quadCLTs, // QuadCLT [] quadCLTs,
dxyzatr_dt[nscene], // double [][] dxyzatr_dt_nscene,
dbg_mb_img, //double [][] dbg_mb_img, // initialize to double[quadCLTs.length][]
mb_max_gain, // double mb_max_gain,
mb_ref_disparity, // double [] mb_ref_disparity,
ref_index, // int ref_index,
nscene, // int nscene,
debug_scene, // = 68 int debug_scene, // = 68
debugLevel); // int debugLevel);
}
if (nscene == ref_index) { // set GPU reference CLT data - should be before other scenes
mb_vectors_ref = null;
if (mb_en) {
// should get velocities from HashMap at reference scene from timestamp , not re-calculate.
// double [][] dxyzatr_dt_ref = getVelocities( // was already set
// quadCLTs, // QuadCLT [] quadCLTs,
// ref_index); // int nscene)
mb_vectors_ref = OpticalFlow.getMotionBlur(
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[ref_index], // QuadCLT scene, // can be the same as ref_scene
......@@ -1415,8 +1536,6 @@ public class Interscene {
scenes_xyzatr[nscene][1],
dxyzatr_dt[nscene][0],
dxyzatr_dt[nscene][1]
// quadCLTs[nscene].getErsCorrection().getErsXYZ_dt(), // same as dxyzatr_dt[nscene][0], just keep for future adjustments?
// quadCLTs[nscene].getErsCorrection().getErsATR_dt() // same as dxyzatr_dt[nscene][1], just keep for future adjustments?
);
} else { // if (nscene == ref_index)
double [][] mb_vectors = null;
......@@ -1459,14 +1578,8 @@ public class Interscene {
}
}
double [][] scene_pull = ErsCorrection.combineXYZATR(scene_xyzatr,diff_pull);
// double [] min_max = new double[3]; // {min, max, actual rms)
// int [] fail_reason = new int[1]; // null or int[1]: 0 - OK, 2 - LMA, 3 - min, 4 - max
if (mb_en) {
// double [][] dxyzatr_dt_scene = getVelocities(
// quadCLTs, // QuadCLT [] quadCLTs,
// nscene); // int nscene)
double [][] dxyzatr_dt_scene = dxyzatr_dt[nscene];
mb_vectors = OpticalFlow.getMotionBlur(
quadCLTs[ref_index], // QuadCLT ref_scene,
......@@ -1614,6 +1727,149 @@ public class Interscene {
return earliest_scene;
}
// Make it the only entry point
// uses *_dt from
// quadCLTs[nscene*].getErsCorrection().getErsXYZATR_dt()
public static double[][] adjustDiffPairsLMAInterscene( // compare two scenes, first may be reference, use motion blur
CLTParameters clt_parameters,
boolean use_lma_dsi,
boolean fpn_disable, // disable fpn filter if images are known to be too close
boolean disable_ers,
double [] min_max, // null or pair of minimal and maximal offsets
int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max
QuadCLT [] quadCLTs,
int ref_index,
TpTask[][] tp_tasks_ref, // Should be TpTask[2][*]. If tp_tasks_ref[0] == null will calculate
// at set first scene to the GPU
double [][] pXpYD_ref, // should be se or at least double [num_tiles][] if tp_tasks_ref[0] == null
// will be recalculated when tp_tasks_ref[0] == null, but for reference frame
int nscene0, // may be == ref_index
int nscene1, // compares to nscene0
double [] ref_disparity, // null or alternative reference disparity
boolean [] reliable_ref, // null or bitmask of reliable reference tiles
double [][] scene0_xyzatr,
double [][] scene1_xyzatr,
double [][] scene1_xyzatr_pull,
boolean[] param_select,
double [] param_regweights,
double [] rms_out, // null or double [2]
double max_rms,
boolean mb_en,
double mb_tau, // 0.008; // time constant, sec
double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
int debugLevel)
{
// boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
if (ref_disparity == null) {
ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0];
}
// TODO: set reference as new version of adjustPairsLMAInterscene() assumes it set
if ((tp_tasks_ref == null) || (tp_tasks_ref[0] == null)) { // calculate and setup reference (for comparison) scene to GPU
if (tp_tasks_ref == null) {
tp_tasks_ref = new TpTask[2][];
}
int margin = clt_parameters.imp.margin;
double [][] pXpYD_ref1 = OpticalFlow.transformToScenePxPyD( // full size - [tilesX*tilesY], some nulls
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
quadCLTs[ref_index]); // final QuadCLT reference_QuadClt)
if (pXpYD_ref == null) {
pXpYD_ref = pXpYD_ref1;
} else {
System.arraycopy(pXpYD_ref1, 0, pXpYD_ref, 0, pXpYD_ref1.length);
}
double [][] pXpYD_scene0 =pXpYD_ref;
if (nscene0 != ref_index) { // same as in sfm
pXpYD_scene0 = OpticalFlow.transformToScenePxPyD( // will be null for disparity == NaN, total size - tilesX*tilesY
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
scene0_xyzatr[0], // final double [] scene_xyz, // camera center in world coordinates
scene0_xyzatr[1], // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene0], // final QuadCLT scene_QuadClt,
quadCLTs[ref_index]); // final QuadCLT reference_QuadClt)
}
double [][] mb_vectors_scene0 = null;
if (mb_en) {
double [][] scene0_xyzatr_dt = QuadCLTCPU.scaleDtFromErs(
clt_parameters,
quadCLTs[nscene0].getErsCorrection().getErsXYZATR_dt());
mb_vectors_scene0 = OpticalFlow.getMotionBlur( // same as in sfm
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[nscene0], // QuadCLT scene, // can be the same as ref_scene
pXpYD_ref, // double [][] ref_pXpYD, // here it is scene, not reference!
scene0_xyzatr[0], // double [] camera_xyz,
scene0_xyzatr[1], // double [] camera_atr,
scene0_xyzatr_dt[0], // double [] camera_xyz_dt,
scene0_xyzatr_dt[1], // double [] camera_atr_dt,
0, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
}
TpTask [][] tp_tasks_ref2 = setReferenceGPU (
clt_parameters, // CLTParameters clt_parameters,
quadCLTs[nscene0], // QuadCLT ref_scene,
ref_disparity,// null // double [] ref_disparity, // null or alternative reference disparity
pXpYD_scene0, // double [][] ref_pXpYD,
reliable_ref, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors_scene0, // double [][] mb_vectors, // now [2][ntiles];
debugLevel); // int debug_level)
System.arraycopy(tp_tasks_ref2,0, tp_tasks_ref, 0, tp_tasks_ref2.length);
}
double [][] mb_vectors_scene1 = null;
if (mb_en) {
double [][] scene1_xyzatr_dt = QuadCLTCPU.scaleDtFromErs(
clt_parameters,
quadCLTs[nscene1].getErsCorrection().getErsXYZATR_dt());
mb_vectors_scene1 = OpticalFlow.getMotionBlur(
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[nscene1], // QuadCLT scene, // can be the same as ref_scene
pXpYD_ref, // double [][] ref_pXpYD, // here it is scene, not reference!
scene1_xyzatr[0], // double [] camera_xyz,
scene1_xyzatr[1], // double [] camera_atr,
scene1_xyzatr_dt[0], // double [] camera_xyz_dt,
scene1_xyzatr_dt[1], // double [] camera_atr_dt,
0, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
}
return adjustPairsLMAInterscene( // assumes reference GPU data is already set - once for multiple scenes
clt_parameters, // CLTParameters clt_parameters,
true, // boolean initial_adjust,
false, // boolean fpn_disable, // disable fpn filter if images are known to be too close
disable_ers, // boolean disable_ers,
min_max, // double [] min_max, // null or pair of minimal and maximal offsets
fail_reason, // int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max
quadCLTs[nscene0], // QuadCLT reference_QuadClt,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
tp_tasks_ref[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
quadCLTs[nscene1], // QuadCLT scene_QuadClt,
scene1_xyzatr[0], // double [] camera_xyz,
scene1_xyzatr[1], // double [] camera_atr,
scene1_xyzatr_pull[0], // double [] scene_xyz_pull, // if both are not null, specify target values to pull to
scene1_xyzatr_pull[1], // double [] scene_atr_pull,
param_select, // boolean[] param_select,
param_regweights, // double [] param_regweights,
rms_out, // double [] rms_out, // null or double [2]
max_rms, // double max_rms,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors_scene1, // double [][] mb_vectors, // now [2][ntiles];
debugLevel); // int debug_level)
}
public static double[][] adjustPairsLMAInterscene(
CLTParameters clt_parameters,
boolean use_lma_dsi,
......@@ -1676,7 +1932,6 @@ public class Interscene {
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
tp_tasks_ref2[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
scene_QuadClt, // QuadCLT scene_QuadClt,
camera_xyz, // double [] camera_xyz,
camera_atr, // double [] camera_atr,
......@@ -1695,12 +1950,12 @@ public class Interscene {
public static double[][] adjustPairsLMAInterscene( // assumes reference scene already set in GPU.
CLTParameters clt_parameters,
boolean initial_adjust,
boolean initial_adjust, // will remember adjustment parameters for comparison
boolean fpn_disable, // disable fpn filter if images are known to be too close
boolean disable_ers,
double [] min_max, // null or pair of minimal and maximal offsets
int [] fail_reason, // null or int[1]: 0 - OK, 1 - LMA, 2 - min, 3 - max
QuadCLT reference_QuadClt,
QuadCLT reference_QuadClt, // First in comparison pair, does not need to be reference?
double [] ref_disparity, // null or alternative reference disparity
double [][] pXpYD_ref, // pXpYD for the reference scene
boolean [] reliable_ref, // null or bitmask of reliable reference tiles
......@@ -1721,6 +1976,7 @@ public class Interscene {
int debug_level)
{
boolean show_initial = true; // debug feature
// boolean debug_corr_fpn = false;
boolean use3D = clt_parameters.ilp.ilma_3d;
boolean filter_by_ers = clt_parameters.ilp.ilma_ers_quads;
double gap_frac = clt_parameters.ilp.ilma_gap_frac ; // 0.25;
......@@ -1740,7 +1996,7 @@ public class Interscene {
double[] camera_atr0 = camera_atr.clone();
double [][][] coord_motion = null;
boolean show_corr_fpn = (!clt_parameters.multiseq_run) && (debug_level > -1); // -3; *********** Change to debug FPN correleation ***
// float [][] dbg_corr_fpn = debug_corr_fpn ? (new float [34][]) : null;
int nlma = 0;
// TODO: save ers_scene.ers_watr_center_dt and ers_scene.ers_wxyz_center_dt before first run lma, restore on failure
// it is saved to backup parameters when first lma run (nlma == 0), // boolean first_run,
......@@ -1766,7 +2022,7 @@ public class Interscene {
margin, // final int margin,
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
facc_2d_img, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
null, // final float [][] dbg_corr_fpn,
null, // dbg_corr_fpn, // final float [][] dbg_corr_fpn,
near_important, // boolean near_important, // do not reduce weight of the near tiles
false, // boolean all_fpn, // do not lower thresholds for non-fpn (used during search)
initial_adjust, // boolean initial_adjust,
......@@ -1926,7 +2182,7 @@ public class Interscene {
}
if (show_corr_fpn) { // repeat after last adjustment to get images
// int num_components = intersceneLma.getNumComponents();
int num_components = 2; // herte - no disparity
int num_components = 2; // here - no disparity
String [] fpn_dbg_titles = new String[num_components + scene_QuadClt.getNumSensors() * 2];
fpn_dbg_titles[0] = "X_avg";
fpn_dbg_titles[1] = "Y_avg";
......@@ -2644,6 +2900,10 @@ public class Interscene {
if (fail_reason != null) {
fail_reason[0]=FAIL_REASON_MIN;
}
if (debug_level > -3){
System.out.println ("interCorrPair(): avg_offs = "+avg_offs+
" < "+min_max[0]+", sw = "+sw);
}
return null;
}
......
src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java
View file @ defa789f
......@@ -4803,6 +4803,16 @@ public class OpticalFlow {
// consider copyJP4src for the lower half (now it is not needed)
}
force_initial_orientations = true;
} else {
try_ref_scene.restoreInterProperties( // restore properties for interscene processing (extrinsics, ers, ...) // get relative poses (98)
null, // String path, // full name with extension or null to use x3d directory
false, // boolean all_properties,// null, // Properties properties, // if null - will only save extrinsics)
debugLevel);
first_last_index = try_ref_scene.getFirstLastIndex(quadCLTs);
if (first_last_index == null) {
// if ((first_last_index == null) ||(first_last_index[0] < 0) || (first_last_index[1] < 0)) {
force_initial_orientations = true;
}
}
}
} else {
......@@ -4929,6 +4939,26 @@ public class OpticalFlow {
int center_index =quadCLTs[last_index].getReferenceIndex(null);
if (center_index == -1) {
force_initial_orientations = true;
} else {
QuadCLT try_ref_scene = (QuadCLT) quadCLT_main.spawnNoModelQuadCLT( // will conditionImageSet
quadCLTs[last_index].getReferenceTimestamp(), // set_channels[last_index].set_name,
clt_parameters,
colorProcParameters, //
threadsMax,
debugLevel-2);
if (try_ref_scene == null) {
force_initial_orientations = true;
} else {
try_ref_scene.restoreInterProperties( // restore properties for interscene processing (extrinsics, ers, ...) // get relative poses (98)
null, // String path, // full name with extension or null to use x3d directory
false, // boolean all_properties,// null, // Properties properties, // if null - will only save extrinsics)
debugLevel);
int [] first_last_index = try_ref_scene.getFirstLastIndex(quadCLTs);
if ((first_last_index == null)) { // ||(first_last_index[0] < 0) || (first_last_index[1] < 0)) {
force_initial_orientations = true;
}
}
}
} else {
......
src/main/java/com/elphel/imagej/tileprocessor/QuadCLTCPU.java
View file @ defa789f
......@@ -414,6 +414,97 @@ public class QuadCLTCPU {
return z_avg;
}
/**
* Estimate average pixel offset between 2 scenes for avoiding FPN. This scene
* should have DSI to estimate average distance
* @param xyzatr0 - first scene [[x,y,z],[a,t,r]]
* @param xyzatr1 - second scene [[x,y,z],[a,t,r]]
* @param use_lma
* @return
*/
public double estimatePixelShift(
double [][] xyzatr0,
double [][] xyzatr1,
boolean use_lma
) {
return estimatePixelShift(
xyzatr0,
xyzatr1,
use_lma,
0);
}
public double estimatePixelShift(
double [][] xyzatr0,
double [][] xyzatr1,
boolean use_lma,
int mode
) {
boolean sign_x = (mode & 1) != 0;
boolean sign_y = (mode & 2) != 0;
double z = getAverageZ(use_lma);
if (Double.isNaN(z)) {
System.out.println("estimatePixelShift(): failed estimating distance - no DSI or too few DSI tiles.");
return Double.NaN;
}
double aztl_pix_per_rad = 1.0/getErsCorrection().getCorrVector().getTiltAzPerPixel();
double roll_pix_per_rad = 1.0/getErsCorrection().getCorrVector().getRollPixel();
double [][] dxyzatr = new double [2][3];
for (int i = 0; i<2;i++) for (int j=0; j < 3; j++) {
dxyzatr[i][j] = xyzatr1[i][j]-xyzatr0[i][j];
}
double eff_az = dxyzatr[1][0] + (sign_x?-1:1) * dxyzatr[0][0] / z;
double eff_tl = dxyzatr[1][1] + (sign_y?-1:1) * dxyzatr[0][1] / z;
double pix_x = -eff_az*aztl_pix_per_rad;
double pix_y = eff_tl*aztl_pix_per_rad;
double pix_roll = dxyzatr[1][2] * roll_pix_per_rad;
double pix_zoom = (0.25*getErsCorrection().getSensorWH()[0])* dxyzatr[0][2]/z;
double est_pix = Math.sqrt(pix_x*pix_x + pix_y*pix_y + pix_roll*pix_roll + pix_zoom*pix_zoom);
return est_pix;
}
public double estimateAverageShift(
double [][] xyzatr0,
double [][] xyzatr1,
double average_z,
boolean use_rot,
boolean rectilinear
) {
ErsCorrection ers = getErsCorrection();
double disp_avg = ers.getDisparityFromZ(average_z);
int [] wh = ers.getSensorWH();
double [][] xy_pairs = {{0.5*wh[0], 0.5*wh[1]}};
if (use_rot) {
xy_pairs = new double [][] {
{0.25*wh[0],0.25*wh[1]},
{0.75*wh[0],0.25*wh[1]},
{0.25*wh[0],0.75*wh[1]},
{0.75*wh[0],0.75*wh[1]}};
}
double s2 = 0.0;
for (double [] xy:xy_pairs) {
double [] pXpYD = ers.getImageCoordinatesERS(
null, // QuadCLT cameraQuadCLT, // camera station that got image to be to be matched
xy[0], // double px, // pixel coordinate X in the reference view
xy[1], // double py, // pixel coordinate Y in the reference view
disp_avg, // double disparity, // this reference disparity
!rectilinear, // boolean distortedView, // This camera view is distorted (diff.rect), false - rectilinear
xyzatr0[0], // double [] reference_xyz, // this view position in world coordinates (typically zero3)
xyzatr0[1], // double [] reference_atr, // this view orientation relative to world frame (typically zero3)
!rectilinear, // boolean distortedCamera, // camera view is distorted (false - rectilinear)
xyzatr1[0], // double [] camera_xyz, // camera center in world coordinates
xyzatr1[1], // double [] camera_atr, // camera orientation relative to world frame
OpticalFlow.LINE_ERR); // double line_err); // threshold error in scan lines (1.0)
double dx = pXpYD[0]-xy[0];
double dy = pXpYD[1]-xy[1];
s2 += dx*dx+dy*dy;
}
double offs_avg = Math.sqrt(s2/xy_pairs.length);
return offs_avg;
}
public double [][] getGround(
CLTParameters clt_parameters,
boolean use_lma,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment