Commit 7d6fb681 authored by Andrey Filippov's avatar Andrey Filippov

First version of MB correction, changing format

parent e4c6d901
......@@ -3719,7 +3719,6 @@ public class GpuQuad{ // quad camera description
//change to fixed 511?
final int task_code = ((1 << num_pairs)-1) << GPUTileProcessor.TASK_CORR_BITS; // correlation only
final double min_px = margin;
// final double max_px = img_width - 1 - margin;
final double max_px = geometryCorrection.getSensorWH()[0] - 1 - margin; // sensor width here, not window width
final double [] min_py = new double[num_cams] ;
final double [] max_py = new double[num_cams] ;
......@@ -3727,7 +3726,6 @@ public class GpuQuad{ // quad camera description
min_py [i] = margin + (calcPortsCoordinatesAndDerivatives? geometryCorrection.getWOITops()[i] : 0);
// camera_heights array is only set during conditionImageSet(), not called by the intersceneAccumulate()
// That was correct, as all scenes should be conditioned
// max_py [i] = geometryCorrection.getWOITops()[i] + geometryCorrection.getCameraHeights()[i] - 1 - margin;
max_py [i] = geometryCorrection.getSensorWH()[1] - 1 - margin; // same for all channels?
//.getSensorWH()[0]
}
......@@ -3807,11 +3805,167 @@ public class GpuQuad{ // quad camera description
public static TpTask[][] setInterTasksMotionBlur(
final int num_cams,
final int img_width, // should match pXpYD
final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
final boolean [] selection, // may be null, if not null do not process unselected tiles
// motion blur compensation
final double mb_tau, // 0.008; // time constant, sec
final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
final double [][] mb_vectors, //
final GeometryCorrection geometryCorrection,
final double disparity_corr,
final int margin, // do not use tiles if their centers are closer to the edges
final boolean [] valid_tiles,
final int threadsMax) // maximal number of threads to launch
{
int num_pairs = Correlation2d.getNumPairs(num_cams);
//change to fixed 511?
final int task_code = ((1 << num_pairs)-1) << GPUTileProcessor.TASK_CORR_BITS; // correlation only
final double min_px = margin;
final double max_px = geometryCorrection.getSensorWH()[0] - 1 - margin; // sensor width here, not window width
final double [] min_py = new double[num_cams] ;
final double [] max_py = new double[num_cams] ;
for (int i = 0; i < num_cams; i++) {
min_py [i] = margin + (calcPortsCoordinatesAndDerivatives? geometryCorrection.getWOITops()[i] : 0);
// camera_heights array is only set during conditionImageSet(), not called by the intersceneAccumulate()
// That was correct, as all scenes should be conditioned
max_py [i] = geometryCorrection.getSensorWH()[1] - 1 - margin; // same for all channels?
//.getSensorWH()[0]
}
if (valid_tiles!=null) {
Arrays.fill(valid_tiles, false);
}
final int tilesX = img_width / GPUTileProcessor.DTT_SIZE;
final int tiles = pXpYD.length;
final Matrix [] corr_rots = geometryCorrection.getCorrVector().getRotMatrices(); // get array of per-sensor rotation matrices
final int quad_main = (geometryCorrection != null)? num_cams:0;
final Thread[] threads = ImageDtt.newThreadArray(threadsMax);
final AtomicInteger ai = new AtomicInteger(00);
final AtomicInteger aTiles = new AtomicInteger(0);
final TpTask[][] tp_tasks = new TpTask[2][tiles]; // aTiles.get()]; // [0] - main, [1] - shifted
final double mb_len_scale = -Math.log(1.0 - 1.0/mb_max_gain);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
@Override
public void run() {
for (int nTile = ai.getAndIncrement(); nTile < tiles; nTile = ai.getAndIncrement())
if ((pXpYD[nTile] != null) && (mb_vectors[nTile] != null) && ((selection == null) || selection[nTile])) {
int tileY = nTile / tilesX;
int tileX = nTile % tilesX;
TpTask tp_task = new TpTask(num_cams, tileX, tileY);
TpTask tp_task_sub = new TpTask(num_cams, tileX, tileY);
tp_task.task = task_code;
tp_task_sub.task = task_code;
double disparity = pXpYD[nTile][2] + disparity_corr;
tp_task.target_disparity = (float) disparity; // will it be used?
tp_task_sub.target_disparity = tp_task.target_disparity; // will it be used?
double [] centerXY = pXpYD[nTile];
tp_task.setCenterXY(centerXY); // this pair of coordinates will be used by GPU to set tp_task.xy and task.disp_dist!
// calculate offset for the secondary tile and weigh
double dx = mb_vectors[nTile][0];
double dy = mb_vectors[nTile][1];
double mb_len = Math.sqrt(dx*dx+dy*dy); // in pixels/s
dx /= mb_len; // unit vector
dy /= mb_len;
mb_len *= mb_tau; // now in pixels
double mb_offs = 1.0; // try 1 pixel. Maybe adjust for non-ortho, e.g. sqrt(2) for diagonal?
double min_offs = mb_len_scale * mb_len;
if (mb_offs < min_offs) {
mb_offs = min_offs;
}
dx *= mb_offs;
dy *= mb_offs;
double [] centerXY_sub = {centerXY[0]+dx,centerXY[1]+dy};
tp_task_sub.setCenterXY(centerXY_sub);
double exp_offs = Math.exp(-mb_offs/mb_len);
double gain = 1.0/(1.0 - exp_offs);
double gain_sub = -gain * exp_offs;
tp_task.setScale(gain);
tp_task_sub.setScale(gain_sub);
boolean bad_margins = false;
if (calcPortsCoordinatesAndDerivatives) { // for non-GPU?
double [][] disp_dist = new double[quad_main][]; // used to correct 3D correlations (not yet used here)
double [][] centersXY_main = geometryCorrection.getPortsCoordinatesAndDerivatives(
geometryCorrection, // GeometryCorrection gc_main,
false, // boolean use_rig_offsets,
corr_rots, // Matrix [] rots,
null, // Matrix [][] deriv_rots,
null, // double [][] pXYderiv, // if not null, should be double[8][]
disp_dist, // used to correct 3D correlations
centerXY[0],
centerXY[1],
disparity); // + disparity_corr);
tp_task.setDispDist(disp_dist);
tp_task.xy = new float [centersXY_main.length][2];
for (int i = 0; i < centersXY_main.length; i++) {
if ( (centersXY_main[i][0] < min_px) || (centersXY_main[i][0] > max_px) ||
(centersXY_main[i][1] < min_py[i]) || (centersXY_main[i][1] > max_py[i])) {
bad_margins = true;
break;
}
tp_task.xy[i][0] = (float) centersXY_main[i][0];
tp_task.xy[i][1] = (float) centersXY_main[i][1];
}
// same for the second entry
double [][] disp_dist_sub = new double[quad_main][]; // used to correct 3D correlations (not yet used here)
double [][] centersXY_main_sub = geometryCorrection.getPortsCoordinatesAndDerivatives(
geometryCorrection, // GeometryCorrection gc_main,
false, // boolean use_rig_offsets,
corr_rots, // Matrix [] rots,
null, // Matrix [][] deriv_rots,
null, // double [][] pXYderiv, // if not null, should be double[8][]
disp_dist_sub, // used to correct 3D correlations
centerXY_sub[0],
centerXY_sub[1],
disparity); // + disparity_corr);
tp_task_sub.setDispDist(disp_dist);
tp_task_sub.xy = new float [centersXY_main.length][2];
for (int i = 0; i < centersXY_main.length; i++) {
if ( (centersXY_main[i][0] < min_px) || (centersXY_main[i][0] > max_px) ||
(centersXY_main[i][1] < min_py[i]) || (centersXY_main[i][1] > max_py[i])) {
bad_margins = true;
break;
}
tp_task_sub.xy[i][0] = (float) centersXY_main_sub[i][0];
tp_task_sub.xy[i][1] = (float) centersXY_main_sub[i][1];
}
} else { // only check center for margins
if ( (centerXY[0] < min_px) || (centerXY[0] > max_px) ||
(centerXY[1] < min_py[0]) || (centerXY[1] > max_py[0]) ||
(centerXY_sub[0] < min_px) || (centerXY_sub[0] > max_px) ||
(centerXY_sub[1] < min_py[0]) || (centerXY_sub[1] > max_py[0])) {
bad_margins = true;
// break;
}
}
if (bad_margins) {
continue;
}
int tp_task_index = aTiles.getAndIncrement();
tp_tasks[0][tp_task_index] = tp_task;
tp_tasks[1][tp_task_index] = tp_task_sub;
if (valid_tiles!=null) {
valid_tiles[nTile] = true;
}
}
}
};
}
ImageDtt.startAndJoin(threads);
final TpTask[][] tp_tasks_out = new TpTask[2][aTiles.get()];
System.arraycopy(tp_tasks[0], 0, tp_tasks_out[0], 0, tp_tasks_out[0].length);
System.arraycopy(tp_tasks[1], 0, tp_tasks_out[1], 0, tp_tasks_out[1].length);
return tp_tasks_out;
}
public void setLpfRbg(
float [][] lpf_rbg, // 4 64-el. arrays: r,b,g,m
......
......@@ -11,13 +11,28 @@ public class TpTask {
public float[][] xy_aux = null;
public float [][] disp_dist = null;
// public float weight;
public float scale = 0.0f; // for motion blur correction
// 0.0 - set (as it was). >0 multiply and set. <0 multiply and accumulate
public static int getSize(int num_sensors) {
return 5 + 2* num_sensors + 4 * num_sensors;
// return 5 + 2* num_sensors + 4 * num_sensors;
return 6 + 2* num_sensors + 4 * num_sensors; // added scale
}
public int getSize() {
return 5 + 2* num_sensors + 4 * num_sensors;
// return 5 + 2* num_sensors + 4 * num_sensors;
return getSize(num_sensors);
}
public void setScale(float scale) {
this.scale = scale;
}
public void setScale(double scale) {
this.scale = (float) scale;
}
public float getScale() {
return scale;
}
public TpTask(
int num_sensors,
......@@ -54,6 +69,7 @@ public class TpTask {
target_disparity = flt[indx++]; // 2
centerXY[0] = flt[indx++]; // 3
centerXY[1] = flt[indx++]; // 4
scale = flt[indx++]; // 5
if (use_aux) {
xy_aux = new float[num_sensors][2];
for (int i = 0; i < num_sensors; i++) {
......@@ -165,7 +181,7 @@ public class TpTask {
flt[indx++] = this.target_disparity; // 2
flt[indx++] = centerXY[0]; // 3
flt[indx++] = centerXY[1]; // 4
flt[indx++] = scale; // 5
float [][] offsets = use_aux? this.xy_aux: this.xy;
for (int i = 0; i < num_sensors; i++) {
if (offsets != null) {
......
......@@ -1199,12 +1199,56 @@ public class ImageDtt extends ImageDttCPU {
gpuQuad.updateTasks(
tp_tasks,
false); // boolean use_aux // while is it in class member? - just to be able to free
// Skipping if ((fdisp_dist != null) || (fpxpy != null)) {...
// int [] wh = null;
// int erase_clt = 1; // NaN;
gpuQuad.execConvertDirect(use_reference_buffer, wh, erase_clt); // put results into a "reference" buffer
}
public void setReferenceTDMotionBlur(
final int erase_clt,
final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
final boolean use_reference_buffer,
final TpTask[][] tp_tasks,
final double gpu_sigma_r, // 0.9, 1.1
final double gpu_sigma_b, // 0.9, 1.1
final double gpu_sigma_g, // 0.6, 0.7
final double gpu_sigma_m, // = 0.4; // 0.7;
final int threadsMax, // maximal number of threads to launch
final int globalDebugLevel)
{
final float [][] lpf_rgb = new float[][] {
floatGetCltLpfFd(gpu_sigma_r),
floatGetCltLpfFd(gpu_sigma_b),
floatGetCltLpfFd(gpu_sigma_g),
floatGetCltLpfFd(gpu_sigma_m)
};
gpuQuad.setLpfRbg( // constants memory - same for all cameras
lpf_rgb,
globalDebugLevel > 2);
gpuQuad.setTasks( // copy tp_tasks to the GPU memory
tp_tasks[0], // TpTask [] tile_tasks,
false, // use_aux); // boolean use_aux)
imgdtt_params.gpu_verify); // boolean verify
// Why always NON-UNIFORM grid? Already set in tp_tasks
gpuQuad.execSetTilesOffsets(false); // false); // prepare tiles offsets in GPU memory, using NON-UNIFORM grid (pre-calculated)
// update tp_tasks
gpuQuad.updateTasks(
tp_tasks[0],
false); // boolean use_aux // while is it in class member? - just to be able to free
gpuQuad.execConvertDirect(use_reference_buffer, wh, erase_clt); // put results into a "reference" buffer
// second tasks (subtracting MB)
gpuQuad.setTasks( // copy tp_tasks to the GPU memory
tp_tasks[1], // TpTask [] tile_tasks,
false, // use_aux); // boolean use_aux)
imgdtt_params.gpu_verify); // boolean verify
// Why always NON-UNIFORM grid? Already set in tp_tasks
gpuQuad.execSetTilesOffsets(false); // false); // prepare tiles offsets in GPU memory, using NON-UNIFORM grid (pre-calculated)
// update tp_tasks
gpuQuad.updateTasks(
tp_tasks[1],
false); // boolean use_aux // while is it in class member? - just to be able to free
gpuQuad.execConvertDirect(use_reference_buffer, wh, -1); // erase_clt); // put results into a "reference" buffer
}
......
......@@ -43,6 +43,12 @@ public class IntersceneLma {
this.thread_invariant = thread_invariant;
this.opticalFlow = opticalFlow;
}
public double [][] getLastJT(){
return last_jt;
}
public double[] getLastRms() {
return last_rms;
}
......@@ -171,19 +177,9 @@ public class IntersceneLma {
final int debug_level)
{
scenesCLT = new QuadCLT [] {reference_QuadClt, scene_QuadClt};
// this.vector_XYS = vector_XYS;
par_mask = param_select;
macrotile_centers = centers;
num_samples = 2 * centers.length;
/*
for (int i = 0; i < vector_XYS.length; i++){
if (((vector_XYS[i] == null) && (centers[i]!=null)) ||
((vector_XYS[i] != null) && (centers[i]==null))) {
vector_XYS[i] = null;
centers[i]= null;
}
}
*/
ErsCorrection ers_ref = reference_QuadClt.getErsCorrection();
ErsCorrection ers_scene = scene_QuadClt.getErsCorrection();
final double [] scene_xyz = (scene_xyz0 != null) ? scene_xyz0 : ers_scene.camera_xyz;
......@@ -201,19 +197,22 @@ public class IntersceneLma {
scene_atr[0], scene_atr[1], scene_atr[2],
scene_xyz[0], scene_xyz[1], scene_xyz[2]};
parameters_full = full_parameters_vector.clone();
if (first_run || (backup_parameters_full == null)) {
if ((vector_XYS != null) && (first_run || (backup_parameters_full == null))) {
backup_parameters_full = full_parameters_vector.clone();
}
int num_pars = 0;
for (int i = 0; i < par_mask.length; i++) if (par_mask[i]) num_pars++;
par_indices = new int [num_pars];
num_pars = 00;
num_pars = 0;
for (int i = 0; i < par_mask.length; i++) if (par_mask[i]) par_indices[num_pars++] = i;
parameters_vector = new double [par_indices.length];
for (int i = 0; i < par_indices.length; i++) parameters_vector[i] = full_parameters_vector[par_indices[i]];
// parameters_initial = parameters_vector.clone();
if (vector_XYS != null) {// skip when used for the motion blur vectors, not LMA
setSamplesWeights(vector_XYS); // not regularized yet !
} else {
weights = null; // new double[2 * centers.length];
}
last_jt = new double [parameters_vector.length][];
if (debug_level > 1) {
......@@ -225,6 +224,10 @@ public class IntersceneLma {
scenesCLT[1], // final QuadCLT scene_QuadClt,
scenesCLT[0], // final QuadCLT reference_QuadClt,
debug_level); // final int debug_level)
if (vector_XYS == null) {
return; // for MB vectors (noLMA)
}
double [][] wjtj = getWJtJlambda( // USED in lwir all NAN
0.0, // final double lambda,
last_jt); // final double [][] jt) all 0???
......@@ -727,10 +730,12 @@ public class IntersceneLma {
final double [] scene_atr = new double[3];
final double [] reference_xyz = new double[3]; // will stay 0
final double [] reference_atr = new double[3]; // will stay 0
final double [] fx = new double [weights.length];
final boolean mb_mode = (weights == null);
final int weights_length = mb_mode ? (2 * macrotile_centers.length) : weights.length;
final double [] fx = mb_mode ? null : (new double [weights_length]); // weights.length]; : weights.length :
if (jt != null) {
for (int i = 0; i < jt.length; i++) {
jt[i] = new double [weights.length];
jt[i] = new double [weights_length]; // weights.length];
}
}
......@@ -758,14 +763,13 @@ public class IntersceneLma {
scene_atr, // double [] atr);
false)[0]; // boolean invert));
// double [][][] derivs = new double [macrotile_centers.length + parameters_vector.length][][];
final Thread[] threads = ImageDtt.newThreadArray(opticalFlow.threadsMax);
final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int iMTile = ai.getAndIncrement(); iMTile < macrotile_centers.length; iMTile = ai.getAndIncrement()) {
if ((macrotile_centers[iMTile]!=null) &&(weights[2*iMTile] > 0.0)){ // was: weights[iMTile]?
if ((macrotile_centers[iMTile]!=null) && (mb_mode || (weights[2*iMTile] > 0.0))){ // was: weights[iMTile]?
//infinity_disparity
boolean is_infinity = macrotile_centers[iMTile][2] < infinity_disparity;
double [][] deriv_params = ers_ref.getDPxSceneDParameters(
......@@ -782,8 +786,10 @@ public class IntersceneLma {
if (deriv_params!= null) {
boolean bad_tile = false;
if (!bad_tile) {
if (!mb_mode) {
fx[2 * iMTile + 0] = deriv_params[0][0]; // pX
fx[2 * iMTile + 1] = deriv_params[0][1]; // pY
}
if (jt != null) {
for (int i = 0; i < par_indices.length; i++) {
int indx = par_indices[i] + 1;
......@@ -792,6 +798,11 @@ public class IntersceneLma {
}
}
}
} else if (mb_mode) {
for (int i = 0; i < par_indices.length; i++) {
jt[i][2 * iMTile + 0] = Double.NaN; // pX
jt[i][2 * iMTile + 1] = Double.NaN; // ; // pY (disparity is not used)
}
}
}
}
......@@ -799,6 +810,9 @@ public class IntersceneLma {
};
}
ImageDtt.startAndJoin(threads);
if (mb_mode) {
return null;
}
// pull to the initial parameter values
for (int i = 0; i < par_indices.length; i++) {
fx [i + 2 * macrotile_centers.length] = vector[i]; // - parameters_initial[i]; // scale will be combined with weights
......
......@@ -238,6 +238,11 @@ public class IntersceneMatchParameters {
public double eq_weight_scale = 20.0; // maximal boost ratio
public double eq_level = 0.9; // equalization level (0.0 - leave as is, 1.0 - boost to have the same supertile strength as average)
public boolean mb_en = true;
public double mb_tau = 0.008; // time constant, sec
public double mb_max_gain = 5.0; // motion blur maximal gain (if more - move second point more than a pixel
public boolean stereo_merge = true;
public int stereo_gap = 32; // pixels between right and left frames
public double stereo_intereye = 63.5; // mm
......@@ -710,8 +715,15 @@ public class IntersceneMatchParameters {
gd.addNumericField("Equalization level", this.eq_level, 5,7,"",
"Target supertile strength will be set to: 0 - original strength (no modification), 1.0 - average supertile strength.");
gd.addTab("Stereo/Video","Parameters for stereo video generation");
gd.addTab("MB","Motion Blur");
gd.addCheckbox ("Compensate motion blur", this.mb_en,
"Ebable motion blur correction.");
gd.addNumericField("Sensor time constant", this.mb_tau, 5,7,"s",
"Sensor exponential decay in seconds.");
gd.addNumericField("Maximal gain", this.mb_max_gain, 5,7,"x",
"Maximal gain for motion blur correction (if needed more for 1 pixel, increase offset).");
gd.addTab("Stereo/Video","Parameters for stereo video generation");
gd.addMessage ("Stereo");
if (stereo_views.length > 0) {
String [] stereo_choices = new String [stereo_views.length + 1];
......@@ -989,7 +1001,9 @@ public class IntersceneMatchParameters {
this.eq_weight_scale = gd.getNextNumber();
this.eq_level = gd.getNextNumber();
this.mb_en = gd.getNextBoolean();
this.mb_tau = gd.getNextNumber();
this.mb_max_gain = gd.getNextNumber();
if (stereo_views.length > 0) {
int i = gd.getNextChoiceIndex();
......@@ -1260,6 +1274,10 @@ public class IntersceneMatchParameters {
properties.setProperty(prefix+"eq_weight_scale", this.eq_weight_scale+""); // double
properties.setProperty(prefix+"eq_level", this.eq_level+""); // double
properties.setProperty(prefix+"mb_en", this.mb_en+""); // boolean
properties.setProperty(prefix+"mb_tau", this.mb_tau+""); // double
properties.setProperty(prefix+"mb_max_gain", this.mb_max_gain+""); // double
properties.setProperty(prefix+"stereo_merge", this.stereo_merge+""); // boolean
properties.setProperty(prefix+"stereo_gap", this.stereo_gap+""); // int
properties.setProperty(prefix+"stereo_intereye", this.stereo_intereye+""); // double
......@@ -1484,6 +1502,10 @@ public class IntersceneMatchParameters {
if (properties.getProperty(prefix+"eq_weight_scale")!=null) this.eq_weight_scale=Double.parseDouble(properties.getProperty(prefix+"eq_weight_scale"));
if (properties.getProperty(prefix+"eq_level")!=null) this.eq_level=Double.parseDouble(properties.getProperty(prefix+"eq_level"));
if (properties.getProperty(prefix+"mb_en")!=null) this.mb_en=Boolean.parseBoolean(properties.getProperty(prefix+"mb_en"));
if (properties.getProperty(prefix+"mb_tau")!=null) this.mb_tau=Double.parseDouble(properties.getProperty(prefix+"mb_tau"));
if (properties.getProperty(prefix+"mb_max_gain")!=null) this.mb_max_gain=Double.parseDouble(properties.getProperty(prefix+"mb_max_gain"));
if (properties.getProperty(prefix+"stereo_merge")!=null) this.stereo_merge=Boolean.parseBoolean(properties.getProperty(prefix+"stereo_merge"));
if (properties.getProperty(prefix+"stereo_gap")!=null) this.stereo_gap=Integer.parseInt(properties.getProperty(prefix+"stereo_gap"));
if (properties.getProperty(prefix+"stereo_intereye")!=null) this.stereo_intereye=Double.parseDouble(properties.getProperty(prefix+"stereo_intereye"));
......@@ -1724,6 +1746,10 @@ public class IntersceneMatchParameters {
imp.eq_weight_scale = this.eq_weight_scale;
imp.eq_level = this.eq_level;
imp.mb_en = this.mb_en;
imp.mb_tau = this.mb_tau;
imp.mb_max_gain = this.mb_max_gain;
imp.stereo_merge = this.stereo_merge;
imp.stereo_gap = this.stereo_gap;
imp.stereo_intereye = this. stereo_intereye;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment