Skip to content

I
imagej-elphel
Commit 256632ef authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

MB correction for inter-scene correlations

parent ff4e1319
Hide whitespace changes
Inline Side-by-side
Showing with 1051 additions and 181 deletions
src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java
View file @ 256632ef
......@@ -5935,42 +5935,6 @@ public class OpticalFlow {
ImagePlus imp_scene = null;
double [][] dxyzatr_dt = null;
if (mb_en) {
/*
get_velocities:
{
int nscene0 = nscene - 1;
if ((nscene0 < 0) ||
(quadCLTs[nscene0]== null)||
(ers_reference.getSceneXYZ(quadCLTs[nscene0].getImageName())== null) ||
(ers_reference.getSceneATR(quadCLTs[nscene0].getImageName())== null)) {
nscene0 = nscene;
}
int nscene1 = nscene + 1;
if ((nscene1 > ref_index) || (quadCLTs[nscene1]== null)) {
nscene1 = nscene;
}
if (nscene1 == nscene0) {
System.out.println("**** Isoloated scene!!! skipping... now may only happen for a ref_scene****");
break get_velocities;
}
double dt = quadCLTs[nscene1].getTimeStamp() - quadCLTs[nscene0].getTimeStamp();
String ts0 = quadCLTs[nscene0].getImageName();
String ts1 = quadCLTs[nscene1].getImageName();
double [] scene_xyz0 = ers_reference.getSceneXYZ(ts0);
double [] scene_atr0 = ers_reference.getSceneATR(ts0);
if (scene_xyz0 == null) {
System.out.println ("BUG: No egomotion data for timestamp "+ts0);
break get_velocities;
}
double [] scene_xyz1 = (nscene1== ref_index)? ZERO3:ers_reference.getSceneXYZ(ts1);
double [] scene_atr1 = (nscene1== ref_index)? ZERO3:ers_reference.getSceneATR(ts1);
dxyzatr_dt = new double[2][3];
for (int i = 0; i < 3; i++) {
dxyzatr_dt[0][i] = (scene_xyz1[i]-scene_xyz0[i])/dt;
dxyzatr_dt[1][i] = (scene_atr1[i]-scene_atr0[i])/dt;
}
}
*/
dxyzatr_dt = getVelocities(
quadCLTs, // QuadCLT [] quadCLTs,
nscene); // int nscene)
......@@ -6153,7 +6117,7 @@ public class OpticalFlow {
System.out.println("buildSeries(): trying adjustPairsLMA() with initial offset azimuth: "+
atr[0]+", tilt ="+atr[1]);
}
double [][][] coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
double [][][] coord_motion = interCorrPair_old( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
null, // double [] ref_disparity, // null or alternative reference disparity
......@@ -12255,7 +12219,133 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
return null;
}
public double [][][] interCorrPair( // return [tilesX*telesY]{ref_pXpYD, dXdYS}
/**
* Prepare and set GPU reference TD data to be used for interscene correlatiuon. Optionally uses
* motion blur correction (if mb_vectors != null)
* @param clt_parameters general parameters
* @param ref_scene reference scene QuadCLT instance
* @param ref_disparity either alternative disparity array or null to use it from the reference scene itself
* @param ref_pXpYD - precalculated or null (will be calculated)
* @param selection optional selection to ignore unselected tiles)
* @param margin do not use tiles with centers closer than this to the edges. Measured in pixels.
* @param mb_tau Sensor time constant in seconds (only needed if mb_vectors != null)
* @param mb_max_gain maximal gain fro MB correction (higher MB corrected by increasing offset)
* @param mb_vectors [2][tiles] {dx/dt[tiles],dx/dt[tiles]} motion blur vectors or null
* @param debug_level debug level
* @return TpTask[][] arrays used to program GPU. With no MB has only one element, with MB - two
*/
public static TpTask[][] setReferenceGPU (
CLTParameters clt_parameters,
QuadCLT ref_scene,
double [] ref_disparity, // null or alternative reference disparity
double [][] ref_pXpYD,
final boolean [] selection, // may be null, if not null do not process unselected tiles
final int margin,
// motion blur compensation
double mb_tau, // 0.008; // time constant, sec
double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
double [][] mb_vectors, // now [2][ntiles];
int debug_level)
{
if (!ref_scene.hasGPU()) {
throw new IllegalArgumentException ("setReferenceGPU(): CPU mode not supported");
}
boolean toRGB = clt_parameters.imp.toRGB ; // true;
int erase_clt = (toRGB? clt_parameters.imp.show_color_nan : clt_parameters.imp.show_mono_nan) ? 1:0;
boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
if (ref_scene.getGPU() != null) {
ref_scene.getGPU().setGpu_debug_level(debug_level - 4); // monitor GPU ops >=-1
}
final double disparity_corr = 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
//ref_disparity
if (ref_disparity == null) {
ref_disparity = ref_scene.getDLS()[use_lma_dsi?1:0];
}
if (ref_pXpYD == null) {
ref_pXpYD = transformToScenePxPyD( // full size - [tilesX*tilesY], some nulls
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
ref_scene, // final QuadCLT scene_QuadClt,
ref_scene, // final QuadCLT reference_QuadClt)
QuadCLT.THREADS_MAX);
}
ImageDtt image_dtt = new ImageDtt(
ref_scene.getNumSensors(),
clt_parameters.transform_size,
clt_parameters.img_dtt,
ref_scene.isAux(),
ref_scene.isMonochrome(),
ref_scene.isLwir(),
clt_parameters.getScaleStrength(ref_scene.isAux()),
ref_scene.getGPU());
TpTask[][] tp_tasks_ref;
if (mb_vectors!=null) {
tp_tasks_ref = GpuQuad.setInterTasksMotionBlur( // "true" reference, with stereo actual reference will be offset
ref_scene.getNumSensors(),
ref_scene.getErsCorrection().getSensorWH()[0],
!ref_scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
ref_pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
selection, // final boolean [] selection, // may be null, if not null do not process unselected tiles
// motion blur compensation
mb_tau, // final double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, //final double [][] mb_vectors, //
ref_scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
disparity_corr, // final double disparity_corr,
margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
QuadCLT.THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
ref_scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
image_dtt.setReferenceTDMotionBlur( // tp_tasks_ref will be updated
erase_clt,
null, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
true, // final boolean use_reference_buffer,
tp_tasks_ref, // final TpTask[] tp_tasks,
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
QuadCLT.THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
debug_level); // final int globalDebugLevel);
} else {
tp_tasks_ref = new TpTask[1][];
tp_tasks_ref[0] = GpuQuad.setInterTasks( // "true" reference, with stereo actual reference will be offset
ref_scene.getNumSensors(),
ref_scene.getErsCorrection().getSensorWH()[0],
!ref_scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
ref_pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
selection, // final boolean [] selection, // may be null, if not null do not process unselected tiles
ref_scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
disparity_corr, // final double disparity_corr,
margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
QuadCLT.THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
ref_scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
image_dtt.setReferenceTD( // tp_tasks_ref will be updated
erase_clt,
null, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
true, // final boolean use_reference_buffer,
tp_tasks_ref[0], // final TpTask[] tp_tasks,
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
QuadCLT.THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
debug_level); // final int globalDebugLevel);
}
ref_scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
return tp_tasks_ref;
}
@Deprecated
public double [][][] interCorrPair_old( // return [tilesX*telesY]{ref_pXpYD, dXdYS}
CLTParameters clt_parameters,
QuadCLT ref_scene,
double [] ref_disparity, // null or alternative reference disparity
......@@ -12304,7 +12394,8 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
double eq_weight_add = clt_parameters.imp.eq_weight_add; // 0.05;
double eq_weight_scale = clt_parameters.imp.eq_weight_scale; // 10;
double eq_level = clt_parameters.imp.eq_level; // 0.8; // equalize to (log) fraction of average/this strength
if (scene_is_ref_test) {
scene_xyz = ZERO3.clone();
scene_atr = ZERO3.clone();
......@@ -12713,124 +12804,583 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
return coord_motion;
}
/**
* Equalize weights of the motion vectors to boost that of important buy weak one.
* Process overlapping (by half, using shifted cosine weight function) supertiles
* independetly and if it qualifies, increase its tiles weights equlaizing (with
* certain limitations) total supertile weights.
* @param coord_motion [2][tilesX*tilesY][3] input/output arrays.[0][tile][] is
* pXpYD triplet (
* @param stride_hor half of a supertile width
* @param stride_vert half of a supertile height
* @param min_stile_weight minimal total weight of the tiles in a supertile (lower
* will not be modified)
* @param min_stile_number minimal number of defined tiles in a supertile
* @param min_stile_fraction minimal total tile strength compared to the average one
* @param min_disparity minimal disparity of tiles to consider (after filtering)
* @param max_disparity maximal disparity of tiles to consider (after filtering)
* @param weight_add add to each tile after scaling (if total weight < average)
* @param weight_scale scale each tile (if total weight < average)
* @param eq_level equalize (log scale) to this ratio between average and this
* strength (0.0 - leave as is, equalize weak to average strength
* If total new weight of a supertile exceeds average - scale each tile to match. If
* lower - keep as is. Only after this step remove tiles (replace with original weight)
* that are discarded by the disparity filter. Multiply result by the the window and
* accumulate (in 4 passes to prevent contentions for the same destination array
*/
public void equalizeMotionVectorsWeights(
final double [][][] coord_motion,
final int tilesX,
final int stride_hor,
final int stride_vert,
final double min_stile_weight,
final int min_stile_number,
final double min_stile_fraction,
final double min_disparity,
final double max_disparity,
final double weight_add,
final double weight_scale,
final double eq_level) // equalize to (log) fraction of average/this strength
public double [][][] interCorrPair( // return [tilesX*telesY]{ref_pXpYD, dXdYS}
CLTParameters clt_parameters,
QuadCLT ref_scene,
double [] ref_disparity, // null or alternative reference disparity
double [][] pXpYD_ref, // pXpYD for the reference scene
TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
QuadCLT scene,
double [] scene_xyz,
double [] scene_atr,
final boolean [] selection, // may be null, if not null do not process unselected tiles
final int margin,
final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
final float [][] dbg_corr_fpn,
boolean near_important, // do not reduce weight of the near tiles
boolean all_fpn, // do not lower thresholds for non-fpn (used during search)
// motion blur compensation
double [][] mb_vectors, // now [2][ntiles];
int imp_debug_level, // MANUALLY change to 2 for debug!
int debug_level)
{
final int tiles = coord_motion[0].length;
final int tilesY = tiles/tilesX;
final int stile_width = 2 * stride_hor;
final int stile_height = 2 * stride_vert;
double [] whor = new double [stride_hor];
double [] wvert = new double [stride_vert];
for (int i = 0; i < stride_hor; i++) whor[i] = 0.5 *(1.0 - Math.cos(Math.PI*(i+0.5)/stride_hor));
for (int i = 0; i < stride_vert; i++) wvert[i] = 0.5 *(1.0 - Math.cos(Math.PI*(i+0.5)/stride_vert));
final int stilesX = (tilesX - 1)/stride_hor + 2; // 10 // 9
final int stilesY = (tilesY - 1)/stride_vert + 2; // 8 // 7
final int stiles = stilesX * stilesY;
int indx = 0;
final double[] wind = new double[stile_height*stile_width];
for (int iy = 0; iy < stile_height;iy++) {
int iy1 = (iy >= stride_vert) ? (stile_height - 1 - iy) : iy;
for (int ix = 0; ix < stile_width; ix++) {
int ix1 = (ix >= stride_hor) ? (stile_width - 1 - ix) : ix;
wind[indx++] = wvert[iy1] * whor[ix1];
}
if (!ref_scene.hasGPU()) {
throw new IllegalArgumentException ("interCorrPair(): CPU mode not supported");
}
final double [] stile_weight = new double [stiles];
final int [] stile_number = new int [stiles];
final Thread[] threads = ImageDtt.newThreadArray(threadsMax);
final AtomicInteger ai = new AtomicInteger(0);
final int dbg_stile = -1;
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int sTile = ai.getAndIncrement(); sTile < stiles; sTile = ai.getAndIncrement()) {
if (sTile == dbg_stile) {
System.out.println("normalizeMotionVectorsWeights (): dbg_stile = "+dbg_stile);
}
int sTileX = sTile % stilesX;
int sTileY = sTile / stilesX;
stile_weight[sTile] = 0.0;
int num_tiles = 0; // for partial stiles
for (int iy = 0; iy < stile_height;iy++) {
int tileY = (sTileY - 1) * stride_vert + iy;
if ((tileY < 0) || (tileY >= tilesY)) continue;
for (int ix = 0; ix < stile_width; ix++) {
int tileX = (sTileX - 1) * stride_hor + ix;
if ((tileX < 0) || (tileX >= tilesX)) continue;
num_tiles++; // for partial stiles
int tile = tileX + tilesX*tileY;
if ((coord_motion[1][tile] != null) && !Double.isNaN(coord_motion[0][tile][2])) {
stile_weight[sTile] += coord_motion[1][tile][2];
stile_number[sTile]++;
}
}
}
stile_weight[sTile] *= 1.0 * stile_height * stile_height / num_tiles; // increase for partiaL stiles
stile_number[sTile] *= stile_height*stile_height;
stile_number[sTile] /= num_tiles;
}
}
};
}
ImageDtt.startAndJoin(threads);
double sum_stile_str = 0.0;
int num_stiles_defined = 0;
for (int sTile = 0; sTile < stiles; sTile++) {
if (stile_weight[sTile] > 0) {
num_stiles_defined++;
sum_stile_str+=stile_weight[sTile];
}
TileProcessor tp = ref_scene.getTileProcessor();
// Temporary reusing same ref scene ******
boolean scene_is_ref_test = clt_parameters.imp.scene_is_ref_test; // false; // true;
boolean show_2d_correlations = clt_parameters.imp.show2dCorrelations(imp_debug_level); // true;
boolean show_motion_vectors = clt_parameters.imp.showMotionVectors(imp_debug_level); // true;
boolean show_render_ref = clt_parameters.imp.renderRef(imp_debug_level); // false; //true;
boolean show_render_scene = clt_parameters.imp.renderScene(imp_debug_level); // false; // true;
boolean toRGB = clt_parameters.imp.toRGB ; // true;
boolean show_coord_motion = clt_parameters.imp.showCorrMotion(imp_debug_level); // mae its own
int erase_clt = (toRGB? clt_parameters.imp.show_color_nan : clt_parameters.imp.show_mono_nan) ? 1:0;
boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
boolean mov_en = clt_parameters.imp.mov_en; // true; // enable detection/removal of the moving objects during pose matching
boolean mov_debug_images = clt_parameters.imp.showMovementDetection(imp_debug_level);
int mov_debug_level = clt_parameters.imp.movDebugLevel(imp_debug_level);
boolean fpn_remove = clt_parameters.imp.fpn_remove;
double fpn_max_offset = clt_parameters.imp.fpn_max_offset;
double fpn_radius = clt_parameters.imp.fpn_radius;
boolean fpn_ignore_border = clt_parameters.imp.fpn_ignore_border; // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
boolean eq_debug = false;
boolean eq_en = near_important && clt_parameters.imp.eq_en; // true;// equalize "important" FG tiles for better camera XYZ fitting
int eq_stride_hor = clt_parameters.imp.eq_stride_hor; // 8;
int eq_stride_vert = clt_parameters.imp.eq_stride_vert; // 8;
double eq_min_stile_weight = clt_parameters.imp.eq_min_stile_weight; // 0.2; // 1.0;
int eq_min_stile_number = clt_parameters.imp.eq_min_stile_number; // 10;
double eq_min_stile_fraction =clt_parameters.imp.eq_min_stile_fraction; // 0.02; // 0.05;
double eq_min_disparity = clt_parameters.imp.eq_min_disparity; // 5;
double eq_max_disparity = clt_parameters.imp.eq_max_disparity; // 100;
double eq_weight_add = clt_parameters.imp.eq_weight_add; // 0.05;
double eq_weight_scale = clt_parameters.imp.eq_weight_scale; // 10;
double eq_level = clt_parameters.imp.eq_level; // 0.8; // equalize to (log) fraction of average/this strength
final double gpu_sigma_corr = clt_parameters.getGpuCorrSigma(ref_scene.isMonochrome());
final double gpu_sigma_rb_corr = ref_scene.isMonochrome()? 1.0 : clt_parameters.gpu_sigma_rb_corr;
final double gpu_sigma_log_corr = clt_parameters.getGpuCorrLoGSigma(ref_scene.isMonochrome());
boolean mb_en = clt_parameters.imp.mb_en ;
double mb_tau = clt_parameters.imp.mb_tau; // 0.008; // time constant, sec
double mb_max_gain = clt_parameters.imp.mb_max_gain; // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
int tilesX = tp.getTilesX();
int tilesY = tp.getTilesY();
double [][][] coord_motion = null; // new double [2][tilesX*tilesY][];
final double [][][] motion_vectors = show_motion_vectors?new double [tilesY *tilesX][][]:null;
final float [][][] fclt_corr = ((accum_2d_corr != null) || show_2d_correlations) ?
(new float [tilesX * tilesY][][]) : null;
if (debug_level > 1) {
System.out.println("interCorrPair(): "+IntersceneLma.printNameV3("ATR",scene_atr)+
" "+IntersceneLma.printNameV3("XYZ",scene_xyz));
}
if (num_stiles_defined <=0) {
System.out.println("normalizeMotionVectorsWeights(): no defined supertiles, bailing out");
return;
ImageDtt image_dtt;
image_dtt = new ImageDtt(
numSens,
clt_parameters.transform_size,
clt_parameters.img_dtt,
ref_scene.isAux(),
ref_scene.isMonochrome(),
ref_scene.isLwir(),
clt_parameters.getScaleStrength(ref_scene.isAux()),
ref_scene.getGPU());
if (ref_scene.getGPU() != null) {
ref_scene.getGPU().setGpu_debug_level(debug_level - 4); // monitor GPU ops >=-1
}
final boolean [] dbg_mod = new boolean [stiles];
final double [] tile_new_strength = new double [tiles]; // accumulate new strengths here
final double avg_stile_str = sum_stile_str / num_stiles_defined;
final double min_combo_weight = Math.max(min_stile_weight, min_stile_fraction * avg_stile_str);
for (int offsy = 0; offsy < 2; offsy++) { // avoiding overlap
final int foffsy = offsy;
final int sty2 = (stilesY + 1 - offsy) / 2;
for (int offsx = 0; offsx < 2; offsx++) {
final int foffsx = offsx;
final int stx2 = (stilesX + 1 - offsx) / 2;
final double disparity_corr = 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction);
//ref_disparity
if (ref_disparity == null) {
ref_disparity = ref_scene.getDLS()[use_lma_dsi?1:0];
}
double [][] scene_pXpYD = transformToScenePxPyD( // will be null for disparity == NaN, total size - tilesX*tilesY
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
scene, // final QuadCLT scene_QuadClt,
scene_is_ref_test ? null: ref_scene); // final QuadCLT reference_QuadClt)
TpTask[][] tp_tasks;
if (mb_en && (mb_vectors!=null)) {
tp_tasks = GpuQuad.setInterTasksMotionBlur(
scene.getNumSensors(),
scene.getErsCorrection().getSensorWH()[0],
!scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
scene_pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
selection, // final boolean [] selection, // may be null, if not null do not process unselected tiles
// motion blur compensation
mb_tau, // final double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, // final double [][] mb_vectors, //
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
disparity_corr, // final double disparity_corr,
margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
threadsMax); // final int threadsMax) // maximal number of threads to launch
} else {
tp_tasks = new TpTask[1][];
tp_tasks[0] = GpuQuad.setInterTasks(
scene.getNumSensors(),
scene.getErsCorrection().getSensorWH()[0],
!scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
scene_pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
selection, // final boolean [] selection, // may be null, if not null do not process unselected tiles
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
disparity_corr, // final double disparity_corr,
margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
threadsMax); // final int threadsMax) // maximal number of threads to launch
}
scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
float [][][][] fcorr_td = null; // no accumulation, use data in GPU
if (mb_en && (mb_vectors!=null)) {
image_dtt.interCorrTDMotionBlur(
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
tp_tasks, // final TpTask[][] tp_tasks,
fcorr_td, // final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of 6 corr pairs
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
gpu_sigma_rb_corr, // final double gpu_sigma_rb_corr, // = 0.5; // apply LPF after accumulating R and B correlation before G, monochrome ? 1.0 :
gpu_sigma_corr, // final double gpu_sigma_corr, // = 0.9;gpu_sigma_corr_m
gpu_sigma_log_corr, // final double gpu_sigma_log_corr, // hpf to reduce dynamic range for correlations
clt_parameters.corr_red, // final double corr_red, // +used
clt_parameters.corr_blue, // final double corr_blue,// +used
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
threadsMax, // final int threadsMax, // maximal number of threads to launch
debug_level); // final int globalDebugLevel);
} else {
image_dtt.interCorrTD(
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
tp_tasks[0], // final TpTask[] tp_tasks,
fcorr_td, // final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of 6 corr pairs
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
gpu_sigma_rb_corr, // final double gpu_sigma_rb_corr, // = 0.5; // apply LPF after accumulating R and B correlation before G, monochrome ? 1.0 :
gpu_sigma_corr, // final double gpu_sigma_corr, // = 0.9;gpu_sigma_corr_m
gpu_sigma_log_corr, // final double gpu_sigma_log_corr, // hpf to reduce dynamic range for correlations
clt_parameters.corr_red, // final double corr_red, // +used
clt_parameters.corr_blue, // final double corr_blue,// +used
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
threadsMax, // final int threadsMax, // maximal number of threads to launch
debug_level); // final int globalDebugLevel);
}
if (show_render_scene) { //set toRGB=false
ImagePlus imp_render_scene = scene.renderFromTD (
-1, // final int sensor_mask,
false, // boolean merge_channels,
clt_parameters, // CLTParameters clt_parameters,
clt_parameters.getColorProcParameters(ref_scene.isAux()), //ColorProcParameters colorProcParameters,
clt_parameters.getRGBParameters(), //EyesisCorrectionParameters.RGBParameters rgbParameters,
null, // int [] wh,
toRGB, // boolean toRGB,
false, //boolean use_reference
"GPU-SHIFTED-SCENE"); // String suffix)
imp_render_scene.show();
}
if (dbg_corr_fpn != null) { // 2*16 or 2*17 (average, individual)
float [][] foffsets = getInterCorrOffsetsDebug(
tp_tasks_ref, // final TpTask[] tp_tasks_ref,
tp_tasks[0], // final TpTask[] tp_tasks,
tilesX, // final int tilesX,
tilesY); // final int tilesY
for (int i = 0; (i < dbg_corr_fpn.length) && (i < foffsets.length); i++) {
dbg_corr_fpn[i] = foffsets[i];
}
}
double [][] fpn_offsets = null;
if (fpn_remove) {
fpn_offsets = getInterCorrOffsets(
fpn_max_offset, // final double max_offset,
tp_tasks_ref, // final TpTask[] tp_tasks_ref,
tp_tasks[0], // final TpTask[] tp_tasks,
tilesX, // final int tilesX,
tilesY); // final int tilesY);
}
double half_disparity = near_important ? 0.0 : clt_parameters.imp.half_disparity;
double [][][] dcorr_tiles = (fclt_corr != null)? (new double [tp_tasks[0].length][][]):null;
// will use num_acc with variable number of accumulations (e.g. clusters)
//all_fpn
double min_str = all_fpn ? clt_parameters.imp.min_str_fpn : clt_parameters.imp.min_str;
double min_str_sum = all_fpn ? clt_parameters.imp.min_str_sum_fpn : clt_parameters.imp.min_str_sum;
coord_motion = image_dtt.clt_process_tl_interscene( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
fcorr_td, // final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of all selected corr pairs
null, // float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null). Can be inner null if not used in tp_tasks
null, // double [] dcorr_weight, // alternative to num_acc, compatible with CPU processing (only one non-zero enough)
clt_parameters.gpu_corr_scale, // final double gpu_corr_scale, // 0.75; // reduce GPU-generated correlation values
clt_parameters.getGpuFatZeroInter(ref_scene.isMonochrome()), // final double gpu_fat_zero, // clt_parameters.getGpuFatZero(is_mono);absolute == 30.0
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
// should it be reference or scene? Or any?
tp_tasks[0], // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMA for the integrated correlations
// to be converted to float (may be null)
dcorr_tiles, // final double [][][] dcorr_tiles, // [tile][pair_abs, sparse][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
pXpYD_ref, // ref_pXpYD, // final double [][] pXpYD, // pXpYD for the reference scene
fpn_offsets, // final double [][] fpn_offsets, // null, or per-tile X,Y offset to be blanked
fpn_radius, // final double fpn_radius, // radius to be blanked around FPN offset center
fpn_ignore_border, // final boolean fpn_ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
motion_vectors, // final double [][][] motion_vectors, // [tilesY*tilesX][][] -> [][][num_sel_sensors+1][2]
clt_parameters.imp.run_poly, // final boolean run_poly, // polynomial max, if false - centroid
clt_parameters.imp.use_partial, // final boolean use_partial, // find motion vectors for individual pairs, false - for sum only
clt_parameters.imp.centroid_radius,// final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
clt_parameters.imp.n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
clt_parameters.imp.td_weight, // final double td_weight, // mix correlations accumulated in TD with
clt_parameters.imp.pd_weight, // final double pd_weight, // correlations (post) accumulated in PD
clt_parameters.imp.td_nopd_only, // final boolean td_nopd_only, // only use TD accumulated data if no safe PD is available for the tile.
min_str, // final double min_str_nofpn, // = 0.25;
min_str_sum, // final double min_str_sum_nofpn, // = 0.8; // 5;
clt_parameters.imp.min_str_fpn, // final double min_str, // = 0.25;
clt_parameters.imp.min_str_sum_fpn,// final double min_str_sum, // = 0.8; // 5;
clt_parameters.imp.min_neibs, // final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
clt_parameters.imp.weight_zero_neibs, // final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
half_disparity, // final double half_disparity, // 5.0; // Reduce weight twice for this disparity
clt_parameters.imp.half_avg_diff, // final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice
clt_parameters.tileX, // final int debug_tileX,
clt_parameters.tileY, // final int debug_tileY,
threadsMax, // final int threadsMax, // maximal number of threads to launch
debug_level);
// final int globalDebugLevel);
if (coord_motion == null) {
System.out.println("clt_process_tl_interscene() returned null");
return null;
}
if (eq_en) {
// double eq_weight_add = (min_str * clt_parameters.imp.pd_weight + min_str_sum * clt_parameters.imp.td_weight) /
// (clt_parameters.imp.pd_weight + clt_parameters.imp.td_weight);
double [] strength_backup = null;
if (eq_debug) { // **** Set manually in debugger ****
strength_backup = new double [coord_motion[1].length];
for (int i = 0; i < strength_backup.length; i++) if (coord_motion[1][i] != null) {
strength_backup[i] = coord_motion[1][i][2];
}else {
strength_backup[i] = Double.NaN;
}
}
do {
// restore
if (strength_backup != null) {
for (int i = 0; i < strength_backup.length; i++) if (coord_motion[1][i] != null) {
coord_motion[1][i][2] = strength_backup[i];
}
}
equalizeMotionVectorsWeights(
coord_motion, // final double [][][] coord_motion,
tilesX, // final int tilesX,
eq_stride_hor, // final int stride_hor,
eq_stride_vert, // final int stride_vert,
eq_min_stile_weight, // final double min_stile_weight,
eq_min_stile_number, // final int min_stile_number,
eq_min_stile_fraction, // final double min_stile_fraction,
eq_min_disparity, // final double min_disparity,
eq_max_disparity, // final double max_disparity,
eq_weight_add, // final double weight_add,
eq_weight_scale, // final double weight_scale)
eq_level); // equalize to (log) fraction of average/this strength
if (eq_debug) {
String [] mvTitles = {"dx", "dy","conf", "conf0", "pX", "pY","Disp","defined"}; // ,"blurX","blurY", "blur"};
double [][] dbg_img = new double [mvTitles.length][tilesX*tilesY];
for (int l = 0; l < dbg_img.length; l++) {
Arrays.fill(dbg_img[l], Double.NaN);
}
for (int nTile = 0; nTile < coord_motion[0].length; nTile++) {
if (coord_motion[0][nTile] != null) {
for (int i = 0; i <3; i++) {
dbg_img[4+i][nTile] = coord_motion[0][nTile][i];
}
}
dbg_img[3] = strength_backup;
if (coord_motion[1][nTile] != null) {
for (int i = 0; i <3; i++) {
dbg_img[0+i][nTile] = coord_motion[1][nTile][i];
}
}
dbg_img[7][nTile] = ((coord_motion[0][nTile] != null)?1:0)+((coord_motion[0][nTile] != null)?2:0);
}
(new ShowDoubleFloatArrays()).showArrays( // out of boundary 15
dbg_img,
tilesX,
tilesY,
true,
scene.getImageName()+"-"+ref_scene.getImageName()+"-coord_motion-eq",
mvTitles);
}
} while (eq_debug);
}
if (mov_en) {
String debug_image_name = mov_debug_images ? (scene.getImageName()+"-"+ref_scene.getImageName()+"-movements"): null;
boolean [] move_mask = getMovementMask(
clt_parameters, // CLTParameters clt_parameters,
coord_motion[1], // double [][] motion, // only x,y,w components
tilesX, // int tilesX,
debug_image_name, // String debug_image_name,
mov_debug_level); // int debug_level);
if (move_mask != null) {
for (int nTile=0; nTile < move_mask.length; nTile++) if (move_mask[nTile]) {
coord_motion[1][nTile]= null;
}
}
}
float [][][] fclt_corr1 = ImageDtt.convertFcltCorr( // partial length, matching corr_indices = gpuQuad.getCorrIndices(); // also sets num_corr_tiles
dcorr_tiles, // double [][][] dcorr_tiles,// [tile][sparse, correlation pair][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
fclt_corr); // float [][][] fclt_corr) // new float [tilesX * tilesY][][] or null
if (show_2d_correlations) { // visualize prepare ref_scene correlation data
float [][] dbg_corr_rslt_partial = ImageDtt.corr_partial_dbg( // not used in lwir
fclt_corr1, // final float [][][] fcorr_data, // [tile][pair][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
image_dtt.getGPU().getCorrIndices(), // tp_tasks, // final TpTask [] tp_tasks, //
tilesX, //final int tilesX,
tilesY, //final int tilesX,
2*image_dtt.transform_size - 1, // final int corr_size,
1000, // will be limited by available layersfinal int layers0,
clt_parameters.corr_border_contrast, // final double border_contrast,
threadsMax, // final int threadsMax, // maximal number of threads to launch
debug_level); // final int globalDebugLevel)
String [] titles = new String [dbg_corr_rslt_partial.length]; // dcorr_tiles[0].length];
for (int i = 0; i < titles.length; i++) {
if (i== (titles.length - 1)) {
titles[i] = "sum";
} else {
titles[i] = "sens-"+i;
}
}
// titles.length = 15, corr_rslt_partial.length=16!
(new ShowDoubleFloatArrays()).showArrays( // out of boundary 15
dbg_corr_rslt_partial,
tilesX*(2*image_dtt.transform_size),
tilesY*(2*image_dtt.transform_size),
true,
scene.getImageName()+"-"+ref_scene.getImageName()+"-interscene",
titles);
}
if (motion_vectors != null) {
int num_sens = 0;
int num_rslt = 0;
find_num_layers:
for (int nt = 0; nt <motion_vectors.length; nt++) {
if (motion_vectors[nt] != null) {
num_sens = motion_vectors[nt].length;
for (int i = 0; i < num_sens; i++) {
if (motion_vectors[nt][i] != null) {
num_rslt = motion_vectors[nt][i].length;
break find_num_layers;
}
}
}
}
double [][] dbg_img = new double [num_sens * num_rslt][tilesX*tilesY];
String [] titles = new String [dbg_img.length];
for (int ns = 0; ns < num_sens; ns++) {
String ss = (ns == (num_sens - 1))?"sum":(""+ns);
for (int nr = 0; nr < num_rslt; nr++) {
int indx = ns*num_rslt+nr;
Arrays.fill(dbg_img[indx],Double.NaN);
String sr="";
switch (nr) {
case 0:sr ="X";break;
case 1:sr ="Y";break;
case 2:sr ="S";break;
default:
sr = ""+nr;
}
titles[indx] = ss+":"+sr;
for (int nt = 0; nt <motion_vectors.length; nt++) {
if ((motion_vectors[nt] != null) && (motion_vectors[nt][ns] != null) ) {
dbg_img[indx][nt] = motion_vectors[nt][ns][nr];
}
}
}
}
(new ShowDoubleFloatArrays()).showArrays( // out of boundary 15
dbg_img,
tilesX,
tilesY,
true,
scene.getImageName()+"-"+ref_scene.getImageName()+"-motion_vectors",
titles);
}
if (show_coord_motion) {
//coord_motion
String [] mvTitles = {"dx", "dy", "conf", "pX", "pY","Disp","defined"}; // ,"blurX","blurY", "blur"};
double [][] dbg_img = new double [mvTitles.length][tilesX*tilesY];
for (int l = 0; l < dbg_img.length; l++) {
Arrays.fill(dbg_img[l], Double.NaN);
}
for (int nTile = 0; nTile < coord_motion[0].length; nTile++) {
if (coord_motion[0][nTile] != null) {
for (int i = 0; i <3; i++) {
dbg_img[3+i][nTile] = coord_motion[0][nTile][i];
}
}
if (coord_motion[1][nTile] != null) {
for (int i = 0; i <3; i++) {
dbg_img[0+i][nTile] = coord_motion[1][nTile][i];
}
}
dbg_img[6][nTile] = ((coord_motion[0][nTile] != null)?1:0)+((coord_motion[0][nTile] != null)?2:0);
}
(new ShowDoubleFloatArrays()).showArrays( // out of boundary 15
dbg_img,
tilesX,
tilesY,
true,
scene.getImageName()+"-"+ref_scene.getImageName()+"-coord_motion",
mvTitles);
}
if (debug_level > 0){
int num_defined = 0;
double sum_strength = 0.0;
for (int i = 0; i < coord_motion[1].length; i++) if (coord_motion[1][i] != null){
sum_strength += coord_motion[1][i][2];
num_defined++;
}
System.out.println ("interCorrPair(): num_defined = "+num_defined+
", sum_strength = "+sum_strength+", avg_strength = "+(sum_strength/num_defined));
}
return coord_motion;
}
/**
* Equalize weights of the motion vectors to boost that of important buy weak one.
* Process overlapping (by half, using shifted cosine weight function) supertiles
* independetly and if it qualifies, increase its tiles weights equlaizing (with
* certain limitations) total supertile weights.
* @param coord_motion [2][tilesX*tilesY][3] input/output arrays.[0][tile][] is
* pXpYD triplet (
* @param stride_hor half of a supertile width
* @param stride_vert half of a supertile height
* @param min_stile_weight minimal total weight of the tiles in a supertile (lower
* will not be modified)
* @param min_stile_number minimal number of defined tiles in a supertile
* @param min_stile_fraction minimal total tile strength compared to the average one
* @param min_disparity minimal disparity of tiles to consider (after filtering)
* @param max_disparity maximal disparity of tiles to consider (after filtering)
* @param weight_add add to each tile after scaling (if total weight < average)
* @param weight_scale scale each tile (if total weight < average)
* @param eq_level equalize (log scale) to this ratio between average and this
* strength (0.0 - leave as is, equalize weak to average strength
* If total new weight of a supertile exceeds average - scale each tile to match. If
* lower - keep as is. Only after this step remove tiles (replace with original weight)
* that are discarded by the disparity filter. Multiply result by the the window and
* accumulate (in 4 passes to prevent contentions for the same destination array
*/
public void equalizeMotionVectorsWeights(
final double [][][] coord_motion,
final int tilesX,
final int stride_hor,
final int stride_vert,
final double min_stile_weight,
final int min_stile_number,
final double min_stile_fraction,
final double min_disparity,
final double max_disparity,
final double weight_add,
final double weight_scale,
final double eq_level) // equalize to (log) fraction of average/this strength
{
final int tiles = coord_motion[0].length;
final int tilesY = tiles/tilesX;
final int stile_width = 2 * stride_hor;
final int stile_height = 2 * stride_vert;
double [] whor = new double [stride_hor];
double [] wvert = new double [stride_vert];
for (int i = 0; i < stride_hor; i++) whor[i] = 0.5 *(1.0 - Math.cos(Math.PI*(i+0.5)/stride_hor));
for (int i = 0; i < stride_vert; i++) wvert[i] = 0.5 *(1.0 - Math.cos(Math.PI*(i+0.5)/stride_vert));
final int stilesX = (tilesX - 1)/stride_hor + 2; // 10 // 9
final int stilesY = (tilesY - 1)/stride_vert + 2; // 8 // 7
final int stiles = stilesX * stilesY;
int indx = 0;
final double[] wind = new double[stile_height*stile_width];
for (int iy = 0; iy < stile_height;iy++) {
int iy1 = (iy >= stride_vert) ? (stile_height - 1 - iy) : iy;
for (int ix = 0; ix < stile_width; ix++) {
int ix1 = (ix >= stride_hor) ? (stile_width - 1 - ix) : ix;
wind[indx++] = wvert[iy1] * whor[ix1];
}
}
final double [] stile_weight = new double [stiles];
final int [] stile_number = new int [stiles];
final Thread[] threads = ImageDtt.newThreadArray(threadsMax);
final AtomicInteger ai = new AtomicInteger(0);
final int dbg_stile = -1;
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int sTile = ai.getAndIncrement(); sTile < stiles; sTile = ai.getAndIncrement()) {
if (sTile == dbg_stile) {
System.out.println("normalizeMotionVectorsWeights (): dbg_stile = "+dbg_stile);
}
int sTileX = sTile % stilesX;
int sTileY = sTile / stilesX;
stile_weight[sTile] = 0.0;
int num_tiles = 0; // for partial stiles
for (int iy = 0; iy < stile_height;iy++) {
int tileY = (sTileY - 1) * stride_vert + iy;
if ((tileY < 0) || (tileY >= tilesY)) continue;
for (int ix = 0; ix < stile_width; ix++) {
int tileX = (sTileX - 1) * stride_hor + ix;
if ((tileX < 0) || (tileX >= tilesX)) continue;
num_tiles++; // for partial stiles
int tile = tileX + tilesX*tileY;
if ((coord_motion[1][tile] != null) && !Double.isNaN(coord_motion[0][tile][2])) {
stile_weight[sTile] += coord_motion[1][tile][2];
stile_number[sTile]++;
}
}
}
stile_weight[sTile] *= 1.0 * stile_height * stile_height / num_tiles; // increase for partiaL stiles
stile_number[sTile] *= stile_height*stile_height;
stile_number[sTile] /= num_tiles;
}
}
};
}
ImageDtt.startAndJoin(threads);
double sum_stile_str = 0.0;
int num_stiles_defined = 0;
for (int sTile = 0; sTile < stiles; sTile++) {
if (stile_weight[sTile] > 0) {
num_stiles_defined++;
sum_stile_str+=stile_weight[sTile];
}
}
if (num_stiles_defined <=0) {
System.out.println("normalizeMotionVectorsWeights(): no defined supertiles, bailing out");
return;
}
final boolean [] dbg_mod = new boolean [stiles];
final double [] tile_new_strength = new double [tiles]; // accumulate new strengths here
final double avg_stile_str = sum_stile_str / num_stiles_defined;
final double min_combo_weight = Math.max(min_stile_weight, min_stile_fraction * avg_stile_str);
for (int offsy = 0; offsy < 2; offsy++) { // avoiding overlap
final int foffsy = offsy;
final int sty2 = (stilesY + 1 - offsy) / 2;
for (int offsx = 0; offsx < 2; offsx++) {
final int foffsx = offsx;
final int stx2 = (stilesX + 1 - offsx) / 2;
final int st2 = sty2*stx2;
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
......@@ -13523,7 +14073,8 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
boolean mb_en = clt_parameters.imp.mb_en;
double mb_tau = clt_parameters.imp.mb_tau; // 0.008; // time constant, sec
double mb_max_gain = clt_parameters.imp.mb_max_gain; // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
int margin = clt_parameters.imp.margin;
int earliest_scene = 0;
boolean use_combo_dsi = clt_parameters.imp.use_combo_dsi;
boolean use_lma_dsi = clt_parameters.imp.use_lma_dsi;
......@@ -13563,20 +14114,22 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
double [][] ref_pXpYD = null;
double [][] dbg_mb_img = null;
double [] mb_ref_disparity =null;
// if (test_motion_blur) {
mb_ref_disparity = interscene_ref_disparity;
if (mb_ref_disparity == null) {
mb_ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0];
}
ref_pXpYD = transformToScenePxPyD( // full size - [tilesX*tilesY], some nulls
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
mb_ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
quadCLTs[ref_index]); // final QuadCLT reference_QuadClt)
if (test_motion_blur) {
mb_ref_disparity = interscene_ref_disparity;
if (mb_ref_disparity == null) {
mb_ref_disparity = quadCLTs[ref_index].getDLS()[use_lma_dsi?1:0];
}
ref_pXpYD = transformToScenePxPyD( // full size - [tilesX*tilesY], some nulls
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
mb_ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
quadCLTs[ref_index]); // final QuadCLT reference_QuadClt)
dbg_mb_img = new double[quadCLTs.length][];
dbg_mb_img = new double[quadCLTs.length][];
}
// }
ErsCorrection ers_reference = quadCLTs[ref_index].getErsCorrection();
......@@ -13585,8 +14138,15 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
scenes_xyzatr[ref_index] = new double[2][3]; // all zeros
// should have at least next or previous non-null
int debug_scene = -15;
double maximal_series_rms = 0.00;
double maximal_series_rms = 0.0;
double [][] mb_vectors_ref = null;
TpTask[][] tp_tasks_ref = null;
for (int nscene = ref_index; nscene >= earliest_scene; nscene--) {
if (nscene == debug_scene) {
System.out.println("nscene = "+nscene);
System.out.println("nscene = "+nscene);
}
if ((quadCLTs[nscene] == null) ||
((nscene != ref_index) &&
((ers_reference.getSceneXYZ(quadCLTs[nscene].getImageName())== null) ||
......@@ -13705,22 +14265,62 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
imp_mbc_merged.show();
}
}
// int debug_scene = -15;
if (nscene != ref_index) {
if (nscene == debug_scene) {
System.out.println("nscene = "+nscene);
System.out.println("nscene = "+nscene);
}
if (nscene == ref_index) { // set GPU reference CLT data
mb_vectors_ref = null;
if (mb_en) {
double [][] dxyzatr_dt_ref = getVelocities(
quadCLTs, // QuadCLT [] quadCLTs,
ref_index); // int nscene)
mb_vectors_ref = getMotionBlur(
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[ref_index], // QuadCLT scene, // can be the same as ref_scene
ref_pXpYD, // double [][] ref_pXpYD, // here it is scene, not reference!
ZERO3, // double [] camera_xyz,
ZERO3, // double [] camera_atr,
dxyzatr_dt_ref[0], // double [] camera_xyz_dt,
dxyzatr_dt_ref[1], // double [] camera_atr_dt,
0, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
}
tp_tasks_ref = setReferenceGPU (
clt_parameters, // CLTParameters clt_parameters,
quadCLTs[ref_index], // QuadCLT ref_scene,
mb_ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
ref_pXpYD, // double [][] ref_pXpYD,
reliable_ref, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors_ref, // double [][] mb_vectors, // now [2][ntiles];
debugLevel); // int debugLevel)
} else {
double [][] mb_vectors = null;
scene_xyz_pre = ers_reference.getSceneXYZ(ts);
scene_atr_pre = ers_reference.getSceneATR(ts);
if (mb_en) {
double [][] dxyzatr_dt_scene = getVelocities(
quadCLTs, // QuadCLT [] quadCLTs,
nscene); // int nscene)
mb_vectors = getMotionBlur(
quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[nscene], // QuadCLT scene, // can be the same as ref_scene
ref_pXpYD, // double [][] ref_pXpYD, // here it is scene, not reference!
scene_xyz_pre, // double [] camera_xyz,
scene_atr_pre, // double [] camera_atr,
dxyzatr_dt_scene[0], // double [] camera_xyz_dt,
dxyzatr_dt_scene[1], // double [] camera_atr_dt,
0, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
}
double [] lma_rms = new double[2];
scenes_xyzatr[nscene] = adjustPairsLMAInterscene(
clt_parameters, // CLTParameters clt_parameters,
quadCLTs[ref_index], // QuadCLT reference_QuadCLT,
interscene_ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
ref_pXpYD, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
tp_tasks_ref[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
quadCLTs[nscene], // QuadCLT scene_QuadCLT,
scene_xyz_pre, // xyz
scene_atr_pre, // atr
......@@ -13728,6 +14328,10 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
clt_parameters.ilp.ilma_regularization_weights, // final double [] param_regweights,
lma_rms, // double [] rms, // null or double [2]
clt_parameters.imp.max_rms, // double max_rms,
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, // double [][] mb_vectors, // now [2][ntiles];
clt_parameters.imp.debug_level); // 1); // -1); // int debug_level);
if (scenes_xyzatr[nscene] == null) {
System.out.println("LMA failed at nscene = "+nscene+". Truncating series.");
......@@ -14010,8 +14614,8 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
}
public double[][] adjustPairsLMAInterscene(
@Deprecated
public double[][] adjustPairsLMAInterscene_old(
CLTParameters clt_parameters,
QuadCLT reference_QuadClt,
double [] ref_disparity, // null or alternative reference disparity
......@@ -14043,7 +14647,7 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
for (; nlma < clt_parameters.imp.max_cycles; nlma ++) {
boolean near_important = nlma > 0;
coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
coord_motion = interCorrPair_old( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
......@@ -14107,7 +14711,7 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
fpn_dbg_titles[3 + 2*i] = "Y-"+i;
}
float [][] dbg_corr_fpn = new float [fpn_dbg_titles.length][];
coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
coord_motion = interCorrPair_old( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
......@@ -14144,10 +14748,11 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
fpn_dbg_titles[3 + 2*i] = "Y-"+i;
}
float [][] dbg_corr_fpn = new float [fpn_dbg_titles.length][];
coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
coord_motion = interCorrPair_old( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
// null, // double [][] pXpYD_ref, // pXpYD for the reference scene
scene_QuadClt, // QuadCLT scene_QuadCLT,
camera_xyz0, // xyz
camera_atr0, // pose[1], // atr
......@@ -14181,6 +14786,271 @@ public double[][] correlateIntersceneDebug( // only uses GPU and quad
if (clt_parameters.imp.debug_level > -2) {
String [] lines1 = intersceneLma.printOldNew((clt_parameters.imp.debug_level > -1)); // false); // boolean allvectors)
System.out.println("Adjusted interscene, iteration="+nlma+
", last RMS = "+intersceneLma.getLastRms()[0]+
" (pure RMS = "+intersceneLma.getLastRms()[1]+")"+
", results:");
for (String line : lines1) {
System.out.println(line);
}
// if (intersceneLma.getLastRms()[0] > clt_parameters.imp.max_rms) {
// System.out.println("Interscene adjustment failed, RMS="+
// intersceneLma.getLastRms()[0]+" > "+ clt_parameters.imp.max_rms);
// return null;
// }
}
if ((rms_out != null) && (intersceneLma.getLastRms() != null)) {
rms_out[0] = intersceneLma.getLastRms()[0];
rms_out[1] = intersceneLma.getLastRms()[1];
//if (lmaResult < 0) { last_rms[0]
}
if (max_rms > 0.0) {
if (lmaResult < 0) { // = 0) {
return null;
}
if (!(intersceneLma.getLastRms()[0] <= max_rms)) {
System.out.println("RMS failed: "+intersceneLma.getLastRms()[0]+" >= " + max_rms);
return null;
}
}
return new double [][] {camera_xyz0, camera_atr0};
}
public double[][] adjustPairsLMAInterscene(
CLTParameters clt_parameters,
QuadCLT reference_QuadClt,
double [] ref_disparity, // null or alternative reference disparity
boolean [] reliable_ref, // null or bitmask of reliable reference tiles
QuadCLT scene_QuadClt,
double [] camera_xyz,
double [] camera_atr,
boolean[] param_select,
double [] param_regweights,
double [] rms_out, // null or double [2]
double max_rms,
int debug_level)
{
// TODO: set reference as new version of adjustPairsLMAInterscene() assumes it set
int margin = clt_parameters.imp.margin;
double [][] pXpYD_ref = transformToScenePxPyD( // full size - [tilesX*tilesY], some nulls
null, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // dls[0], // final double [] disparity_ref, // invalid tiles - NaN in disparity (maybe it should not be masked by margins?)
ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
reference_QuadClt, // final QuadCLT scene_QuadClt,
reference_QuadClt); // final QuadCLT reference_QuadClt)
TpTask[][] tp_tasks_ref2 = setReferenceGPU (
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT ref_scene,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
pXpYD_ref, // double [][] ref_pXpYD,
reliable_ref, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin,
// motion blur compensation
0.0, // double mb_tau, // 0.008; // time constant, sec
0.0, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
null, // double [][] mb_vectors, // now [2][ntiles];
debug_level); // int debug_level)
return adjustPairsLMAInterscene( // assumes reference GPU data is already set - once for multiple scenes
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadClt,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
reliable_ref, // boolean [] reliable_ref, // null or bitmask of reliable reference tiles
tp_tasks_ref2[0], // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
scene_QuadClt, // QuadCLT scene_QuadClt,
camera_xyz, // double [] camera_xyz,
camera_atr, // double [] camera_atr,
param_select, // boolean[] param_select,
param_regweights, // double [] param_regweights,
rms_out, // double [] rms_out, // null or double [2]
max_rms, // double max_rms,
// motion blur compensation
0.0, // double mb_tau, // 0.008; // time constant, sec
0.0, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
null, // double [][] mb_vectors, // now [2][ntiles];
debug_level); // int debug_level)
}
public double[][] adjustPairsLMAInterscene( // assumes reference scene already set in GPU.
CLTParameters clt_parameters,
QuadCLT reference_QuadClt,
double [] ref_disparity, // null or alternative reference disparity
double [][] pXpYD_ref, // pXpYD for the reference scene
boolean [] reliable_ref, // null or bitmask of reliable reference tiles
TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
QuadCLT scene_QuadClt,
double [] camera_xyz,
double [] camera_atr,
boolean[] param_select,
double [] param_regweights,
double [] rms_out, // null or double [2]
double max_rms,
// motion blur compensation
double mb_tau, // 0.008; // time constant, sec
double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
double [][] mb_vectors, // now [2][ntiles];
int debug_level)
{
int margin = clt_parameters.imp.margin;
int sensor_mask_inter = clt_parameters.imp.sensor_mask_inter ; //-1;
float [][][] facc_2d_img = new float [1][][];
IntersceneLma intersceneLma = new IntersceneLma(
clt_parameters.ilp.ilma_thread_invariant);
int lmaResult = -1;
boolean last_run = false;
double[] camera_xyz0 = camera_xyz.clone();
double[] camera_atr0 = camera_atr.clone();
double [][][] coord_motion = null;
boolean show_corr_fpn = debug_level > -1; // -3; *********** Change to debug FPN correleation ***
int nlma = 0;
for (; nlma < clt_parameters.imp.max_cycles; nlma ++) {
boolean near_important = nlma > 0;
// FIXME: not re-calculating motion blur. Maybe do that after each parameter update?
// pass dxyzatr_dt, not mb_vectors? Or update velocities too?
coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
tp_tasks_ref, // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
scene_QuadClt, // QuadCLT scene_QuadCLT,
camera_xyz0, // xyz
camera_atr0, // pose[1], // atr
reliable_ref, // ****null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin,
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
facc_2d_img, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
null, // final float [][] dbg_corr_fpn,
near_important, // boolean near_important, // do not reduce weight of the near tiles
false, // boolean all_fpn, // do not lower thresholds for non-fpn (used during search)
mb_vectors, // double [][] mb_vectors, // now [2][ntiles];
clt_parameters.imp.debug_level, // int imp_debug_level,
debug_level); // 1); // -1); // int debug_level);
if (coord_motion == null) {
System.out.println("adjustPairsLMAInterscene() returned null");
return null;
}
intersceneLma.prepareLMA(
camera_xyz0, // final double [] scene_xyz0, // camera center in world coordinates (or null to use instance)
camera_atr0, // final double [] scene_atr0, // camera orientation relative to world frame (or null to use instance)
// reference atr, xyz are considered 0.0
scene_QuadClt, // final QuadCLT scene_QuadClt,
reference_QuadClt, // final QuadCLT reference_QuadClt,
param_select, // final boolean[] param_select,
param_regweights, // final double [] param_regweights,
coord_motion[1], // final double [][] vector_XYS, // optical flow X,Y, confidence obtained from the correlate2DIterate()
coord_motion[0], // final double [][] centers, // macrotile centers (in pixels and average disparities
(nlma == 0), // boolean first_run,
clt_parameters.imp.debug_level); // final int debug_level)
lmaResult = intersceneLma.runLma(
clt_parameters.ilp.ilma_lambda, // double lambda, // 0.1
clt_parameters.ilp.ilma_lambda_scale_good, // double lambda_scale_good,// 0.5
clt_parameters.ilp.ilma_lambda_scale_bad, // double lambda_scale_bad, // 8.0
clt_parameters.ilp.ilma_lambda_max, // double lambda_max, // 100
clt_parameters.ilp.ilma_rms_diff, // double rms_diff, // 0.001
clt_parameters.imp.max_LMA, // int num_iter, // 20
last_run, // boolean last_run,
debug_level); // int debug_level)
if (lmaResult < 0) {
System.out.println("Interscene adjustment failed, lmaResult="+lmaResult+" < 0");
return null;
}
camera_xyz0 = intersceneLma.getSceneXYZ(false); // true for initial values
camera_atr0 = intersceneLma.getSceneATR(false); // true for initial values
double [] diffs_atr = intersceneLma.getV3Diff(ErsCorrection.DP_DSAZ);
double [] diffs_xyz = intersceneLma.getV3Diff(ErsCorrection.DP_DSX);
if ((diffs_atr[0] < clt_parameters.imp.exit_change_atr) &&
(diffs_xyz[0] < clt_parameters.imp.exit_change_xyz)) {
break;
}
}
if (show_corr_fpn && (debug_level > -1)) { // now not needed, restore if needed
String [] fpn_dbg_titles = new String[2 + scene_QuadClt.getNumSensors() * 2];
fpn_dbg_titles[00] = "X_avg";
fpn_dbg_titles[1] = "X_avg";
for (int i = 0; i < scene_QuadClt.getNumSensors(); i++) {
fpn_dbg_titles[2 + 2*i] = "X-"+i;
fpn_dbg_titles[3 + 2*i] = "Y-"+i;
}
float [][] dbg_corr_fpn = new float [fpn_dbg_titles.length][];
coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
tp_tasks_ref, // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
scene_QuadClt, // QuadCLT scene_QuadCLT,
camera_xyz0, // xyz
camera_atr0, // pose[1], // atr
reliable_ref, // ****null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin,
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
facc_2d_img, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
dbg_corr_fpn, // final float [][] dbg_corr_fpn,
true, // boolean near_important, // do not reduce weight of the near tiles
false, // boolean all_fpn, // do not lower thresholds for non-fpn (used during search)
mb_vectors, // double [][] mb_vectors, // now [2][ntiles];
2, // int imp_debug_level,
debug_level); // 1); // -1); // int debug_level);
TileProcessor tp = reference_QuadClt.getTileProcessor();
int tilesX = tp.getTilesX();
int tilesY = tp.getTilesY();
(new ShowDoubleFloatArrays()).showArrays(
dbg_corr_fpn,
tilesX,
tilesY,
true,
scene_QuadClt.getImageName()+"-"+reference_QuadClt.getImageName()+"-CORR-FPN",
fpn_dbg_titles);
}
if (show_corr_fpn) { // repeat after last adjustment to get images
String [] fpn_dbg_titles = new String[2 + scene_QuadClt.getNumSensors() * 2];
fpn_dbg_titles[00] = "X_avg";
fpn_dbg_titles[1] = "X_avg";
for (int i = 0; i < scene_QuadClt.getNumSensors(); i++) {
fpn_dbg_titles[2 + 2*i] = "X-"+i;
fpn_dbg_titles[3 + 2*i] = "Y-"+i;
}
float [][] dbg_corr_fpn = new float [fpn_dbg_titles.length][];
coord_motion = interCorrPair( // new double [tilesY][tilesX][][];
clt_parameters, // CLTParameters clt_parameters,
reference_QuadClt, // QuadCLT reference_QuadCLT,
ref_disparity, // double [] ref_disparity, // null or alternative reference disparity
pXpYD_ref, // double [][] pXpYD_ref, // pXpYD for the reference scene
tp_tasks_ref, // TpTask[] tp_tasks_ref, // only (main if MB correction) tasks for FPN correction
scene_QuadClt, // QuadCLT scene_QuadCLT,
camera_xyz0, // xyz
camera_atr0, // pose[1], // atr
reliable_ref, // ****null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
margin, // final int margin,
sensor_mask_inter, // final int sensor_mask_inter, // The bitmask - which sensors to correlate, -1 - all.
facc_2d_img, // final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)final float [][][] accum_2d_corr, // if [1][][] - return accumulated 2d correlations (all pairs)
dbg_corr_fpn, // final float [][] dbg_corr_fpn,
true, // boolean near_important, // do not reduce weight of the near tiles
false, // boolean all_fpn, // do not lower thresholds for non-fpn (used during search)
mb_vectors, // double [][] mb_vectors, // now [2][ntiles];
2, // int imp_debug_level,
debug_level); // 1); // -1); // int debug_level);
TileProcessor tp = reference_QuadClt.getTileProcessor();
int tilesX = tp.getTilesX();
int tilesY = tp.getTilesY();
(new ShowDoubleFloatArrays()).showArrays(
dbg_corr_fpn,
tilesX,
tilesY,
true,
scene_QuadClt.getImageName()+"-"+reference_QuadClt.getImageName()+"-CORR-FPN",
fpn_dbg_titles);
}
if (clt_parameters.imp.debug_level > -2) {
String [] lines1 = intersceneLma.printOldNew((clt_parameters.imp.debug_level > -1)); // false); // boolean allvectors)
System.out.println("Adjusted interscene, iteration="+nlma+
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment