Commit 477343de authored by Andrey Filippov's avatar Andrey Filippov

Implementing inter-scene matching with normalization by eigenvectors

parent 81d74a44
......@@ -2680,7 +2680,7 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
gd.addNumericField("Video output frame rate", video_fps, 3,7,"fps", "Frame rate of the video.");
gd.addCheckbox ("Compress AVI with JPEG", avi_compress_jpeg, "Use JPEG for AVI compression (false - use raw).");
gd.addNumericField("AVI JPEG quality", aviJpegQuality, 0, 4, "", "AVI JPEG quality if JPEG compression is used.");
gd.addCheckbox ("Convert AVI to WEBM", run_ffmpeg, "Use ffmped to convert intermediate AVI video to WEBM.");
gd.addCheckbox ("Convert AVI to WEBM", run_ffmpeg, "Use ffmpeg to convert intermediate AVI video to WEBM.");
gd.addStringField ("WEBM output extension", video_ext, 5,"WEBM output file extension including dot, normally \".webm\".");
gd.addStringField ("WEBM codec", video_codec, 5,"WEBM codec \"vp8\" or \"vp9\"(vp9 had problems).");
gd.addNumericField("WEBM CRF", video_crf, 0, 4, "", "WEBM compression quality (lower - better, 10 - good).");
......
......@@ -10,6 +10,7 @@ import com.elphel.imagej.common.PolynomialApproximation;
import com.elphel.imagej.common.ShowDoubleFloatArrays;
import com.elphel.imagej.tileprocessor.Corr2dLMA.Sample;
import Jama.EigenvalueDecomposition;
import Jama.Matrix;
/**
......@@ -2506,6 +2507,183 @@ public class Correlation2d {
}
/**
*
* @param data
* @param data_width
* @param abs_min
* @param rel_min
* @param min_peak
* @param eigen_sub_min - when calculating eigenvectors, subtract min from data, false - just skip
* @param fpn_mask
* @param ignore_border
* @param debug_data null or double[1]
* @param debug
* @return {dx,dy,strength, eig_x, eig_y, lambda0, lambda1), [eig_x, eig_y] <-> labmda0, lambda0 < lambda1.
*/
public static double [] getMaxXYCmEig(
double [] data, // will be modified if fpn_mask != null;
int data_width, // = 2 * transform_size - 1;
double abs_min,
double rel_min,
double min_peak,
double eig_sub_frac, // subtract fraction of threshold {eig_min_abs,eig_min_rel} after selecting by them (0 - select only, will have pedestal)
boolean [] fpn_mask,
boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
double [][] debug_data, // null or double [1]
boolean debug)
{
int data_height = data.length/data_width;
int center_xy = (data_width - 1)/2; // = transform_size - 1;
double x0 = center_xy, y0 = center_xy;
int imax= 0;
for (int i= 1; i < data.length;i++) {
if (Double.isNaN(data[i])) {
System.out.println("NaN in getMaxXYCmEig()");
return null;
}
if (data[i] > data[imax]) {
imax = i;
}
}
if (data[imax] < min_peak) {
return null; // too weak;even before fpn filter
}
int ix0 = imax % data_width;
int iy0 = imax / data_width;
x0 = ix0;
y0 = iy0;
//min_peak
// if (fpn_mask != null
if (fpn_mask != null) { // modifies data, returns null if hits fpn
for (int i = 0; i < fpn_mask.length; i++) if (fpn_mask[i]) {
int iy = i / data_width;
int ix = i - iy * data_width;
if (ignore_border) {
if(((ix - ix0) <= 1) && ((ix - ix0) >= -1) && ((iy - iy0) <= 1) && ((iy - iy0) >= -1)) {
return null; // new double[3];
}
}
int ix1 = 2 * ix0 - ix;
if ((ix1 >= 0) && (ix1 < data_width)) {
int iy1 = 2 * iy0 - iy;
if ((iy1 >= 0) && (iy1 < data_height)) {
data[iy1 * data_width + ix1] = 0.0; // zero out symmetrical to fpn mask around integer maximum
}
}
}
// update imax
imax= 0;
for (int i= 1; i < data.length;i++) {
if (data[i] > data[imax]) {
imax = i;
}
}
if (data[imax] < min_peak) {
return null; // too weak after fpn filter
}
}
// create mask of connected to max pixels
double mx = data[imax];
double min_d = Math.min(abs_min, rel_min*mx);
//
double sub_pedestal = min_d * eig_sub_frac;
boolean [] above_threshold = new boolean [data.length];
for (int i = 0; i < data.length; i++) {
above_threshold[i] = data[i] >= min_d;
}
boolean [] en_data = (new TileNeibs(data_width, data_height)).getConnected(
above_threshold, // boolean [] tiles,
ix0, // int seedX,
iy0); // int seedY)
// find centroid
double s0 = 0, sx=0,sy = 0, sx2 = 0, sy2=0, sxy = 0;
for (int iy = 0; iy < data_height; iy++) {
double y = iy - y0;
for (int ix = 0; ix < data_width; ix++) {
int indx = iy * data_width + ix;
if (en_data[indx]) { // assumes d >0, as it is >= min_d
double x = ix - x0;
double d = data[iy * data_width + ix] - sub_pedestal;
s0 += d;
sx += d * x;
sy += d * y;
sx2 += d * x * x;
sy2 += d * y * y;
sxy += d * x * y;
}
}
}
x0 += sx / s0; // relative to top-left
y0 += sy / s0;
/*
double s0 = 0, sx=0,sy = 0;
for (int iy = 0; iy < data_height; iy++) {
double y = iy - y0;
for (int ix = 0; ix < data_width; ix++) {
int indx = iy * data_width + ix;
if (en_data[indx]) { // assumes d >0, as it is >= min_d
double x = ix - x0;
double d = data[iy * data_width + ix] - sub_pedestal;
s0 += d;
sx += d * x;
sy += d * y;
}
}
}
x0 += sx / s0; // relative to top-left
y0 += sy / s0;
double sx2 = 0, sy2=0, sxy = 0;
for (int iy = 0; iy < data_height; iy++) {
double y = iy - y0;
for (int ix = 0; ix < data_width; ix++) {
int indx = iy * data_width + ix;
if (en_data[indx]) { // assumes d >0, as it is >= min_d
double x = ix - x0;
double d = data[iy * data_width + ix] - sub_pedestal;
sx2 += d * x * x;
sy2 += d * y * y;
sxy += d * x * y;
}
}
}
*/
//https://users.cs.utah.edu/~tch/CS4640/resources/A%20geometric%20interpretation%20of%20the%20covariance%20matrix.pdf
double cxx = sx2 - sx * sx / s0, cyy= sy2 - sy * sy / s0, cxy = sxy - sx * sy / s0;
Matrix covar = new Matrix(new double[][] {{cxx, cxy},{cxy,cyy}});
EigenvalueDecomposition eig = covar.eig();
double [] eigval = {eig.getD().get(0, 0),eig.getD().get(1, 1)};
double [][] eigvec = eig.getV().getArray(); // columns - vectors?
int eig_indx = (eigval[0] > eigval[1]) ? 1 : 0;
double [] rslt = {
x0 - center_xy,
y0 - center_xy,
mx,
eigvec[0][eig_indx],
eigvec[1][eig_indx],
eigval[eig_indx],
eigval[1-eig_indx]};
if (debug){
System.out.println("getMaxXYCm() -> "+rslt[0]+":"+rslt[1]+" ("+rslt[2]+
"), eigv0=["+rslt[3]+","+rslt[4]+"], lambda0="+rslt[5]+", lambda1="+rslt[6]);
}
if (debug_data != null) {
debug_data[0] = data.clone();
for (int i = 0; i < data.length; i++) {
if (!en_data[i]) {
debug_data[0][i] = Double.NaN;
}
}
}
return rslt;
}
/**
* Find maximum of the 2d array projected on a specified vector using centroid.
* On the first stage integer maximum is found, then several refining operations
......
......@@ -12,6 +12,8 @@ import com.elphel.imagej.gpu.GpuQuad;
import com.elphel.imagej.gpu.TpTask;
import com.elphel.imagej.orthomosaic.ComboMatch;
import Jama.EigenvalueDecomposition;
import Jama.Matrix;
import ij.ImagePlus;
import jcuda.Pointer;
......@@ -1812,8 +1814,9 @@ public class ImageDtt extends ImageDttCPU {
startAndJoin(threads);
return map_corr_indices;
}
public double [][][] clt_process_tl_interscene( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
// without eigenvectors (use null for double[][] eigen)
public double [][][] clt_process_tl_interscene_noeigen( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
// only used here to keep extra array element for disparity difference
boolean use3D, // generate disparity difference
......@@ -1858,13 +1861,18 @@ public class ImageDtt extends ImageDttCPU {
final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
final double scale_neibs_td, // scale threshold for the transform-domain average maximums
final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones
final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones
/*
final double eigen_min_abs, // 0.05 values below this do not count for covariance
final double eigen_min_rel, // 0.2 values less than this fraction of tile's max do not count for covariance
final double [][] eigen, // null or [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
*/
final int debug_tileX,
final int debug_tileY,
final int threadsMax, // maximal number of threads to launch
final int globalDebugLevel)
{
// final boolean eigen_sub_min = false; // when calculating eigenvectors, subtract min from data, false - just skip
if (this.gpuQuad == null) {
System.out.println("clt_process_tl_interscene(): this.gpuQuad is null, bailing out");
return null;
......@@ -1982,7 +1990,7 @@ public class ImageDtt extends ImageDttCPU {
@Override
public void run() {
int tileY,tileX,nTile; // , chn;
TileNeibs tn = new TileNeibs(tilesX,tilesY);
// TileNeibs tn = new TileNeibs(tilesX,tilesY);
for (int iCorrTile = ai.getAndIncrement(); iCorrTile < num_tiles; iCorrTile = ai.getAndIncrement()) {
nTile = (fcorr_indices[iCorrTile* used_sensors_list.length] >> GPUTileProcessor.CORR_NTILE_SHIFT);
tileY = nTile / tilesX;
......@@ -2135,7 +2143,7 @@ public class ImageDtt extends ImageDttCPU {
}
}
if (td_weight > 0.0) {
if (td_weight > 0.0) { //
mv_td = Correlation2d.getMaxXYCm( // pre-last - sharp (in FD)
corrs[indx_sum_td], // corrs.length-2], // double [] data,
corr_size, // int data_width, // = 2 * transform_size - 1;
......@@ -2191,16 +2199,20 @@ public class ImageDtt extends ImageDttCPU {
if (mv_pd != null) for (int i = 0; i < mv_pd.length; i++) mv[i] += w_pd * mv_pd[i];
if (mv_td != null) for (int i = 0; i < mv_td.length; i++) mv[i] += w_td * mv_td[i];
if (mv_neib != null) for (int i = 0; i < mv_neib.length; i++) mv[i] += w_ntd * mv_neib[i];
if (mv != null) {
if (pXpYD == null) {
coord_motion[0][nTile] = mv;
} else {
if (pXpYD[nTile] != null) { // seems always
coord_motion[0][nTile] = pXpYD[nTile].clone();
coord_motion[1][nTile] = mv;
}
// mv != null here
if (pXpYD == null) {
coord_motion[0][nTile] = mv;
} else {
if (pXpYD[nTile] != null) { // seems always
coord_motion[0][nTile] = pXpYD[nTile].clone();
coord_motion[1][nTile] = mv;
}
}
// Calculate eigenvector/values (if not null)
// incorrect mixing of TD and PD - PD is much weaker, here we'll just mix source values as weights
if (debugTile0) {
System.out.println("clt_process_tl_correlations(): (eigen) tileX="+tileX+", tileY="+tileY+", nTile="+nTile+", nTile="+nTile);
}
}
}
}
......@@ -2208,7 +2220,6 @@ public class ImageDtt extends ImageDttCPU {
};
}
startAndJoin(threads);
ai.set(0);
final int tiles = tilesX * tilesY;
final double [][] mv = coord_motion[coord_motion.length - 1];
......@@ -2308,6 +2319,479 @@ public class ImageDtt extends ImageDttCPU {
startAndJoin(threads);
return coord_motion;
}
public double [][][] clt_process_tl_interscene( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
// only used here to keep extra array element for disparity difference
boolean use3D, // generate disparity difference
boolean use_neibs,
final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of all selected corr pairs
float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null). Can be inner null if not used in tp_tasks
double [] dcorr_weight, // alternative to num_acc, compatible with CPU processing (only one non-zero enough)
final double gpu_corr_scale, // 0.75; // reduce GPU-generated correlation values
final double gpu_fat_zero, // clt_parameters.getGpuFatZero(is_mono);absolute == 30.0
final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMA for the integrated correlations
// to be converted to float (may be null)
final double [][][] dcorr_tiles, // [tile][pair absolute, sparse][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
final double [][] pXpYD, // pXpYD for the reference scene
final double [][] fpn_offsets, // null, or per-tile X,Y offset to be blanked
final double fpn_radius, // radius to be blanked around FPN offset center
final boolean fpn_ignore_border, // NOT used! only if fpn_mask != null - ignore tile if maximum touches fpn_mask
// final double[][][] motion_vectors, // [tilesY*tilesX][][] -> [][num_sel_sensors+1 or 2][3]
final boolean run_poly, // polynomial max, if false - centroid
final boolean use_partial, // find motion vectors for individual pairs, false - for sum only
final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
final int n_recenter, // when cosine window, re-center window this many times
final double td_weight, // mix correlations accumulated in TD with
final double td_neib_weight, // mix correlations accumulated in TD (neibs)
final double pd_weight, // correlations (post) accumulated in PD
final boolean td_nopd_only, // only use TD accumulated data if no safe PD is available for the tile.
final boolean eig_use_neibs, // use correlation from 9 tiles with neibs, if single-tile fails
final boolean eig_remove_neibs, //remove weak (by-neibs) tiles if they have strong (by-single) neighbor
// final double min_str_nofpn, // = 0.25;
final double eig_str_sum_nofpn,// = 0.8; // 5;
final double eig_str_neib_nofpn,// = 0.8; // 5;
// final double min_str_fpn, // = 0.25;
final double eig_str_sum_fpn, // = 0.8; // 5;
final double eig_str_neib_fpn, // = 0.8; // 5;
final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
final double half_disparity, // 5.0; // Reduce weight twice for this disparity
final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice
final boolean neibs_nofpn_only, // consolidate neighbors for non-fpn tiles only!
final boolean redo_both, // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
final double scale_neibs_td, // scale threshold for the transform-domain average maximums
final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones
final double eigen_min_abs, // 0.05 values below this do not count for covariance
final double eigen_min_rel, // 0.2 values less than this fraction of tile's max do not count for covariance
final double eig_sub_frac, // 1.0; // subtract fraction of threshold {eig_min_abs,eig_min_rel} after selecting by them (0 - select only, will have pedestal)
final double [][] eigen, // null or [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
final boolean eigen_debug, //
final int debug_tileX,
final int debug_tileY,
final int threadsMax, // maximal number of threads to launch
final int globalDebugLevel)
{
// final boolean
// final boolean eigen_sub_min = false; // when calculating eigenvectors, subtract min from data, false - just skip
if (this.gpuQuad == null) {
System.out.println("clt_process_tl_interscene(): this.gpuQuad is null, bailing out");
return null;
}
final boolean extra_sum = true; // use sum of pixel-domain correlations (TD have artifacts for low contrast
// - maybe -related to float vs. double - not tested yet . Probably - still FPN with low offset
final int tilesX= gpuQuad.getTilesX(); // width/transform_size;
final int tilesY= gpuQuad.getTilesY(); // final int tilesY=height/transform_size;
final double [][][] coord_motion = new double [(pXpYD != null)?2:1][tilesX * tilesY][];
// not yet used with GPU
// keep for now for mono, find out what do they mean for macro mode
final int corr_size = transform_size * 2 - 1;
final double [][] debug_lma = imgdtt_params.lmamask_dbg? (new double [6][tilesX*tilesY]):null;
if (debug_lma != null) {
for (int i = 0; i < debug_lma.length; i++) {
Arrays.fill(debug_lma[i], Double.NaN);
}
}
final float [][] pfcorr_weights = ((num_acc != null) || (dcorr_weight != null))? new float[1][] : null;
// This version obeys tp_task order and fills fcorr_td gaps (should be none) with zeros.
int [] corr_indices ;
// now it is always null
if (fcorr_td == null) { // used with no accumulation, assume TD correlation data is still in GPU
corr_indices = gpuQuad.getCorrIndices(); // also sets num_corr_tiles
} else { // never
if (num_acc != null) { // version with float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null)
corr_indices = gpuQuad.setCorrTilesTd( // .length = 295866 should set num_corr_tiles!
tp_tasks, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
true, // final boolean inter_mode, // interscene mode
fcorr_td, // final float [][][][] corr_tiles, // [tileY][tileX][pair][4*64]
num_acc, // float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null)
pfcorr_weights); // float [][] pfcorr_weights) // null or one per correlation tile (num_corr_tiles) to divide fat zero2
} else { // version with // double [] dcorr_weight, // [ntile] (or null), compatible with the CPU version
corr_indices = gpuQuad.setCorrTilesTd( // .length = 295866 should set num_corr_tiles!
tp_tasks, // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMW for the integrated correlations
true, // final boolean inter_mode, // interscene mode
fcorr_td, // final float [][][][] corr_tiles, // [tileY][tileX][pair][4*64]
dcorr_weight, // double [] dcorr_weight, // [ntile] (or null)
pfcorr_weights); // float [][] pfcorr_weights) // null or one per correlation tile (num_corr_tiles) to divide fat zero2
}
}
// corr_indices has TD sum slot
final int num_tiles = corr_indices.length / gpuQuad.getSensInter().length; // number of tiles, regardless of correlation slices
//getSensInterNeib(boolean full)
final int [] map_corr_indices = getMapCorr(corr_indices);
double [] neib_weights_od = {0.7, 0.5};
final boolean use_full = use_partial || (dcorr_tiles != null) || !use_neibs; // old version always correlated all sensors
final int [] used_sensors_list = use_neibs ? gpuQuad.getSensInterNeib(use_full) : gpuQuad.getSensInter(); // last is 0xff - sum of channels
if (use_neibs) {
corr_indices = prepNeibCorr( // updates GPU memory to run a single execCorr2D_normalize
use_full, // final boolean use_partial, // find motion vectors for individual pairs, false - for sum only
neib_weights_od, // double [] neib_weights_od, // {orhto, diag}
map_corr_indices, // int [] map_corr_indices_in,
debug_tileX, // final int debug_tileX,
debug_tileY, // final int debug_tileY,
globalDebugLevel); // final int globalDebugLevel)
}
// final int num_used_slices = corr_indices.length / num_tiles;
int dbg_imax = 0;
for (int ii = 1; ii < corr_indices.length; ii++) {
if (corr_indices[ii] > corr_indices[dbg_imax]) {
dbg_imax=ii;
}
}
// System.out.println("dbg_imax = "+dbg_imax+", corr_indices[dbg_imax]="+corr_indices[dbg_imax]+" tile="+((dbg_imax-16)/17));
if (corr_indices.length == 0) {
return null;
}
float [] fcorr_weights = ((num_acc != null) || (dcorr_weight != null))? pfcorr_weights[0] : null;
gpuQuad.execCorr2D_normalize(
false, // boolean combo, // normalize combo correlations (false - per-pair ones)
gpu_fat_zero, // double fat_zero);
fcorr_weights, // fcorr_weights, // float [] fcorr_weights, // null or one per correlation tile (num_corr_tiles) to divide fat zero2
gpu_corr_rad); // int corr_radius
final float [][] fcorr2D = gpuQuad.getCorr2D(gpu_corr_rad); // int corr_rad);
final int corr_length = fcorr2D[0].length;// all correlation tiles have the same size
// final int num_tiles = corr_indices.length / gpuQuad.getSensInter().length; // number of tiles, regardless of correlation slices
// currently execCorr2D_normalize() output has 17 slices for old variant (no neibs) and 18/2 if (use_neibs)
final int extra_len = extra_sum? 1 : 0;
final int extra_len_eig = eigen_debug? 4: 0; // all, all_remain, weak, weak_remain
// final int corrs_len = ((use_partial || use_neibs) ? used_sensors_list.length:1); // without optional extra_len but including GPU sum
final int corrs_len = (use_neibs || use_partial) ? used_sensors_list.length:1; // without optional extra_len but including GPU sum
final int eigen_indx = (extra_len_eig > 0) ? (corrs_len + extra_len):-1;
final int indx_sum_pd = (extra_len > 0) ? corrs_len : -1;
final int indx_sum_td = use_neibs ? (corrs_len -2): (corrs_len -1);
final int indx_sum_td_neib = use_neibs ? (corrs_len -1): -1;
//num_used_slices
final double [][] corr_wnd = Corr2dLMA.getCorrWnd(
transform_size,
imgdtt_params.lma_wnd);
final double [] corr_wnd_inv_limited = (imgdtt_params.lma_min_wnd <= 1.0)? new double [corr_wnd.length * corr_wnd[0].length]: null;
if (corr_wnd_inv_limited != null) {
double inv_pwr = imgdtt_params.lma_wnd_pwr - (imgdtt_params.lma_wnd - 1.0); // compensate for lma_wnd
for (int i = imgdtt_params.lma_hard_marg; i < (corr_wnd.length - imgdtt_params.lma_hard_marg); i++) {
for (int j = imgdtt_params.lma_hard_marg; j < (corr_wnd.length - imgdtt_params.lma_hard_marg); j++) {
corr_wnd_inv_limited[i * (corr_wnd.length) + j] = 1.0/Math.max(Math.pow(corr_wnd[i][j],inv_pwr), imgdtt_params.lma_min_wnd);
}
}
}
final int [] fcorr_indices = corr_indices;
final int [] fpn_indices = use_neibs?
( new int [] {used_sensors_list[used_sensors_list.length-2],used_sensors_list[used_sensors_list.length-1]}) :
(new int [] {used_sensors_list[used_sensors_list.length-1]});
final int [] iCorrTile_index = eigen_debug? (new int [tilesX*tilesY]) : null;// eigen_debug
final Thread[] threads = newThreadArray(threadsMax);
final AtomicInteger ai = new AtomicInteger(0);
final boolean [] used_td = new boolean [tilesX*tilesY]; // this tile had strong enough TD w/o neibs
// all neibs with strong TD around them will be removed
// not using PD at all? always TD, then neibs?
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
@Override
public void run() {
int tileY,tileX,nTile; // , chn;
// TileNeibs tn = new TileNeibs(tilesX,tilesY);
for (int iCorrTile = ai.getAndIncrement(); iCorrTile < num_tiles; iCorrTile = ai.getAndIncrement()) {
nTile = (fcorr_indices[iCorrTile* used_sensors_list.length] >> GPUTileProcessor.CORR_NTILE_SHIFT);
tileY = nTile / tilesX;
tileX = nTile % tilesX;
boolean debugTile0 =(tileX == debug_tileX) && (tileY == debug_tileY) && (globalDebugLevel > 2); // 0);
if (debugTile0) {
System.out.println("clt_process_tl_correlations(): tileX="+tileX+", tileY="+tileY+", nTile="+nTile+", nTile="+nTile);
}
// zero FPN right in the fcorr2D so it will go to both processing and display
boolean [] fpn_mask = null;
double eig_str_sum = eig_str_sum_nofpn; // higher threshold when FPN is possible
double eig_str_neib = eig_str_neib_nofpn; // higher threshold when FPN is possible
boolean is_fpn=false;
if ((fpn_offsets != null) && (fpn_offsets[nTile] != null)) {
double fpn_x = transform_size - 1 - fpn_offsets[nTile][0]; // 0 -> 7.0
double fpn_y = transform_size - 1 - fpn_offsets[nTile][1]; // 0 -> 7.0
int min_x = (int) Math.max(Math.round(fpn_x - fpn_radius),0);
int max_x = (int) Math.min(Math.round(fpn_x + fpn_radius), corr_size-1);
int min_y = (int) Math.max(Math.round(fpn_y - fpn_radius),0);
int max_y = (int) Math.min(Math.round(fpn_y + fpn_radius), corr_size-1);
fpn_mask = new boolean[fcorr2D[0].length]; // fcorr2D_indx].length];
for (int iy = min_y; iy <= max_y; iy++) {
for (int ix = min_x; ix <= max_x; ix++) {
int indx = iy * corr_size + ix;
fpn_mask[indx] = true;
}
}
eig_str_sum = eig_str_sum_fpn;
eig_str_neib = eig_str_neib_fpn;
is_fpn = true;
}
// double [][] corrs = new double [corrs_len + extra_len + extra_len_eig][]; // 1/17/2/18 +(0/1)
double [][] corrs = new double [corrs_len + extra_len][]; // 1/17/2/18 +(0/1)
// int eigen_indx = (extra_len_eig > 0) ? (corrs_len + extra_len + extra_len_eig -1):-1;
// int eigen_indx = (extra_len_eig > 0) ? (corrs_len + extra_len):-1;
// copy correlation tiles from the GPU's floating point arrays
double scale = 1.0/getNumSensors();
if (extra_sum) {
corrs[corrs_len] = new double [corr_length];
}
// copy all preserved, calculate sum of individual sensors correlations?
// !use_neibs - all slices with individual, corrs_len - may be only combo (1) or all 17
// use_neibs - used_sensors_list.length == corrs_len
for (int isens = corrs_len - 1; isens >= 0; isens--) { // 16..0, 0..0, 17..0, 1..0
int nsens = used_sensors_list.length - corrs_len + isens; // 16..0, 16..16, 17..0, 1..0
corrs[isens] = new double[corr_length];
int fcorr2D_indx = iCorrTile * used_sensors_list.length + nsens;
// convert to double and scale - all slices used
for (int i = 0; i < corr_length; i++) {
corrs[isens][i] = gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
}
// calculate PD sum of individual sensors correlations
if (use_partial && extra_sum && (used_sensors_list[nsens] < getNumSensors())) { // only for individual sensors
for (int i = 0; i < corr_length; i++) {
corrs[corrs_len][i] += scale*corrs[isens][i];
}
}
}
// calculate PD sum of individual sensors correlations if they themselves are not preserved
if (!use_partial && extra_sum && use_full) {
scale *= gpu_corr_scale;
for (int nsens = 0; nsens < (used_sensors_list.length - 1); nsens++) if (used_sensors_list[nsens] < 0xfe){
int fcorr2D_indx = iCorrTile * used_sensors_list.length + nsens;
for (int i = 0; i < corr_length; i++) {
corrs[corrs_len][i] += scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
}
}
}
if (is_fpn) {
for (int i = 0; i < corr_length; i++) if (fpn_mask[i]){
corrs[corrs_len - 1][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
if (use_neibs) {
corrs[corrs_len - 2][i] = 0.0;
}
}
}
if (dcorr_tiles != null) { // This will be visualized (only for visualization?)
int index_es = used_sensors_list.length; // last, OK if extra_len==0
//used_sensors_list
dcorr_tiles[iCorrTile] = new double[used_sensors_list.length + extra_len + extra_len_eig][];
if (extra_sum) {
dcorr_tiles[iCorrTile][index_es] = new double[corr_length];
}
for (int nsens = 0; nsens < used_sensors_list.length; nsens++) { // all but sum
int abs_sens = used_sensors_list[nsens]; // should fork for neibs to full (2 elements)
if ((abs_sens < getNumSensors()) && extra_sum) {
int fcorr2D_indx = iCorrTile * used_sensors_list.length + nsens;
for (int i = 0; i < corr_length; i++) {
dcorr_tiles[iCorrTile][index_es][i] += scale * gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
}
}
dcorr_tiles[iCorrTile][nsens] = new double[corr_length];
int fcorr2D_indx = iCorrTile * used_sensors_list.length + nsens;
for (int i = 0; i < corr_length; i++) {
dcorr_tiles[iCorrTile][nsens][i] = gpu_corr_scale * fcorr2D[fcorr2D_indx][i]; // copy one-by-one converting from floats to doubles
}
}
if (is_fpn) {
for (int i = 0; i < corr_length; i++) if (fpn_mask[i]){
dcorr_tiles[iCorrTile][used_sensors_list.length-1][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
if (use_neibs) {
dcorr_tiles[iCorrTile][used_sensors_list.length-2][i] = 0.0; // instead of fcorr2D[fcorr2D_indx][indx] = 0;
}
}
}
}
// now calculate only for composite
// dcorr_tiles[iCorrTile][eigen_indx] = tile data
// calculate weights and sum
// only two cases:
// 1 - TD only (!neib_notd_only)
// 2 - TD, then TD-neibs if failed (neib_notd_only)
double [][] debug_data = ((dcorr_tiles != null) && (eigen_indx >=0)) ? (new double[1][]):null;
double [] stats_mv = Correlation2d.getMaxXYCmEig(
corrs[indx_sum_td], // double [] data, // will be modified if fpn_mask != null;
corr_size, // int data_width, // = 2 * transform_size - 1;
eigen_min_abs, // double abs_min,
eigen_min_rel, // double rel_min,
eig_str_sum, // double min_peak,
eig_sub_frac, // double eig_sub_frac, // subtract fraction of threshold {eig_min_abs,eig_min_rel} after selecting by them (0 - select only, will have pedestal)
fpn_mask, // boolean [] fpn_mask,
false, // boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
debug_data, // double [][] debug_data, // null or double [1]
false); // boolean debug)
used_td[nTile] = stats_mv != null;
if (stats_mv != null) {
stats_mv[2] -= eig_str_sum * scale_neibs_td;
if (stats_mv[2] <= 0) {
stats_mv = null;
}
}
// UPDATE!
if (!used_td[nTile] && eig_use_neibs) { // try neibs
stats_mv = Correlation2d.getMaxXYCmEig(
corrs[indx_sum_td_neib], // double [] data, // will be modified if fpn_mask != null;
corr_size, // int data_width, // = 2 * transform_size - 1;
eigen_min_abs, // double abs_min,
eigen_min_rel, // double rel_min,
eig_str_neib, // double min_peak,
eig_sub_frac, // double eig_sub_frac, // subtract fraction of threshold {eig_min_abs,eig_min_rel} after selecting by them (0 - select only, will have pedestal)
fpn_mask, // boolean [] fpn_mask,
false, // boolean ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
debug_data, // double [][] debug_data, // null or double [1]
false); // boolean debug)
}
if ((debug_data != null) && (debug_data[0] != null)) {
iCorrTile_index[nTile] = iCorrTile;
dcorr_tiles[iCorrTile][eigen_indx+0] = debug_data [0]; // all
dcorr_tiles[iCorrTile][eigen_indx+1] = debug_data [0].clone(); // all remain
if (!used_td[nTile]) {
dcorr_tiles[iCorrTile][eigen_indx+2] = debug_data [0].clone(); // weak
dcorr_tiles[iCorrTile][eigen_indx+3] = debug_data [0].clone(); // weak REMAIN
}
//used_td[nTile]
}
if (stats_mv != null) {
if (eigen != null) {
eigen[nTile] = new double [] {stats_mv[3],stats_mv[4],stats_mv[5],stats_mv[6]};
stats_mv = new double [] {stats_mv[0],stats_mv[1],stats_mv[2],0,0};
}
// mv != null here
if (pXpYD == null) {
coord_motion[0][nTile] = stats_mv;
} else {
if (pXpYD[nTile] != null) { // seems always
coord_motion[0][nTile] = pXpYD[nTile].clone();
coord_motion[1][nTile] = stats_mv;
}
}
}
}
}
};
}
startAndJoin(threads);
final int tiles = tilesX * tilesY;
final double [][] mv = coord_motion[coord_motion.length - 1];
final double [][] pxd = (coord_motion.length>1) ? coord_motion[0] : null;
// Remove weak (only set by neighbors) if it has a strong neighbor, set by a single tile correlation.
if (eig_remove_neibs) {
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
@Override
public void run() {
TileNeibs tn = new TileNeibs(tilesX,tilesY);
for (int nTile = ai.getAndIncrement(); nTile < tiles; nTile = ai.getAndIncrement()) {
if ((mv[nTile] != null) && !used_td[nTile]) { // is weak (by neibs)
for (int dir = 0; dir < TileNeibs.DIRS; dir++) {
int nTile1 = tn.getNeibIndex(nTile, dir);
if ((nTile1 >= 0) && used_td[nTile1]) {
mv[nTile] = null;
if (pxd != null) {
pxd[nTile] = null;
}
if (iCorrTile_index != null) {
int iCorrTile = iCorrTile_index[nTile];
dcorr_tiles[iCorrTile][eigen_indx+1] = null;
dcorr_tiles[iCorrTile][eigen_indx+3] = null;
}
break;
}
}
}
}
}
};
}
startAndJoin(threads);
}
boolean old_filter = false; // true;
// Reduce weight if differs much from average of 8 neighbors, large disparity, remove too few neibs
final double scale_num_neib = ((weight_zero_neibs >= 0) && (weight_zero_neibs < 1.0)) ? (weight_zero_neibs * 8/(1.0 - weight_zero_neibs)): 0.0;
if (old_filter) {
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
@Override
public void run() {
double l2;
TileNeibs tn = new TileNeibs(tilesX,tilesY);
for (int nTile = ai.getAndIncrement(); nTile < tiles; nTile = ai.getAndIncrement()) {
if ((mv[nTile] != null) && (pXpYD[nTile] != null)) {
int num_neibs=0;
double sx=0.0, sy=0.0;
for (int dir = 0; dir < TileNeibs.DIRS; dir++) {
int nTile1 = tn.getNeibIndex(nTile, dir);
if ((nTile1 >= 0) &&
(mv[nTile1] != null) &&
(pXpYD[nTile1] != null) &&
!Double.isNaN(mv[nTile1][2]) &&
!Double.isNaN(mv[nTile1][0]) &&
!Double.isNaN(mv[nTile1][1])){
num_neibs++;
sx += mv[nTile1][0];
sy += mv[nTile1][1];
}
}
if (num_neibs < min_neibs) { // filter by minimal neighbors
mv[nTile][2] = 0;
continue;
}
if ((weight_zero_neibs >= 0) && (weight_zero_neibs < 1.0)) { // scale weight by number of neighbors
mv[nTile][2] *= (num_neibs + scale_num_neib)/(8.0 + scale_num_neib);
}
if (half_disparity > 0.0) { // scale by disparity
mv[nTile][2] *= half_disparity/(half_disparity + pxd[nTile][2]);
}
if ((half_avg_diff > 0.0) &&(num_neibs > 0)) {
double dx = mv[nTile][0] - sx / num_neibs;
double dy = mv[nTile][1] - sy / num_neibs;
l2 = Math.sqrt(dx * dx + dy * dy);
mv[nTile][2] *= half_avg_diff/(half_avg_diff + l2);
}
}
}
}
};
}
startAndJoin(threads);
}
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
@Override
public void run() {
for (int nTile = ai.getAndIncrement(); nTile < tiles; nTile = ai.getAndIncrement()) {
if ((mv[nTile] != null) && (pXpYD[nTile] != null)) {
if (mv[nTile][2] <= 0) {
mv[nTile] = null;
if (pxd != null) {
pxd[nTile] = null;
}
}
}
}
}
};
}
startAndJoin(threads);
return coord_motion;
}
// using most of the ImageDttCPU.clt_process_tl_correlations
public void clt_process_tl_correlations( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
......
......@@ -8950,11 +8950,18 @@ public class ImageDttCPU {
final float [][] fcorr_data_out = new float[layers][]; // [tilesY*tilesX*tile_size*tile_size];
final Thread[] threads = newThreadArray(threadsMax);
final AtomicInteger ai = new AtomicInteger(0);
/*
for (int layer = 0; layer < layers; layer++) if (fcorr_data[flayer][layer] != null){
fcorr_data_out[layer] = new float[tilesY*tilesX*tile_size*tile_size];
Arrays.fill(fcorr_data_out[layer], Float.NaN);
}
*/
// now some tiles may have null for some layers, non-null - for others
for (int layer = 0; layer < layers; layer++){
fcorr_data_out[layer] = new float[tilesY*tilesX*tile_size*tile_size];
Arrays.fill(fcorr_data_out[layer], Float.NaN);
}
final boolean [] used_layers = new boolean[layers];
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
@Override
......@@ -8970,16 +8977,19 @@ public class ImageDttCPU {
tileY = nTile / tilesX;
tileX = nTile % tilesX;
if (fcorr_data[iCorrTile] != null) {
for (int layer = 0; layer < layers; layer++) if (fcorr_data[iCorrTile][layer] != null){ // sparse
for (int i = 0; i < corr_size;i++){
System.arraycopy(
fcorr_data[iCorrTile][layer],
corr_size* i,
fcorr_data_out[layer],
((tileY*tile_size + i) *tilesX + tileX)*tile_size ,
corr_size);
}
if (fcorr_data[iCorrTile] != null) {
for (int layer = 0; layer < layers; layer++) if (fcorr_data[iCorrTile][layer] != null){ // sparse
used_layers[layer] = true;
for (int i = 0; i < corr_size;i++){
if (fcorr_data[iCorrTile][layer] != null) {
System.arraycopy(
fcorr_data[iCorrTile][layer],
corr_size* i,
fcorr_data_out[layer],
((tileY*tile_size + i) *tilesX + tileX)*tile_size ,
corr_size);
}
}
}
}
}
......@@ -8987,7 +8997,14 @@ public class ImageDttCPU {
};
}
startAndJoin(threads);
for (int layer = 0; layer < layers; layer++){
if (!used_layers[layer]) {
fcorr_data_out[layer]=null;
}
}
return fcorr_data_out;
// remove unused output layers:
}
......
......@@ -3192,7 +3192,7 @@ public class Interscene {
}
boolean [] param_select_mod = param_select;
if (ers_lma && filter_by_ers && !disable_ers) {
double [][] quad_strengths = getQuadStrengths(
double [][] quad_strengths = getQuadStrengths(// uses coord_motion[1][][]
coord_motion, // double [][][] coord_motion,
gap_frac, //double gap_frac, // 0.25
first_QuadClt.getTileProcessor().getTilesX()); // int tilesX);
......@@ -3232,6 +3232,9 @@ public class Interscene {
}
// double [][] ref_xyzatr_inv = ErsCorrection.invertXYZATR(ref_xyzatr);
// TODO: save ers_scene.ers_watr_center_dt and ers_scene.ers_wxyz_center_dt
double [][] eigen = (coord_motion.length < 3)? null: coord_motion[2];
double eig_max_sqrt=clt_parameters.imp.eig_max_sqrt;
double eig_min_sqrt=clt_parameters.imp.eig_min_sqrt;
intersceneLma.prepareLMA(
scene_xyzatr0, // final double [] scene_xyzatr0, // camera center in world coordinates (or null to use instance)
scene_xyzatr_pull, // final double [] scene_xyz_pull, // if both are not null, specify target values to pull to
......@@ -3241,6 +3244,9 @@ public class Interscene {
first_QuadClt, // final QuadCLT reference_QuadClt,
param_select_mod, // param_select, // final boolean[] param_select,
param_regweights, // final double [] param_regweights,
eig_max_sqrt, // final double eig_max_sqrt, // 10; // for sqrt(lambda) - consider infinity (infinite linear feature, a line)
eig_min_sqrt, // final double eig_min_sqrt, // 1; // for sqrt(lambda) - consider infinity (infinite linear feature, a line)
eigen, // final double [][] eigen, // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
coord_motion[1], // final double [][] vector_XYS, // optical flow X,Y, confidence obtained from the correlate2DIterate()
coord_motion[0], // final double [][] centers, // macrotile centers (in pixels and average disparities
(nlma == 0), // boolean first_run,
......@@ -4111,9 +4117,15 @@ public class Interscene {
double [][][] dcorr_tiles = (fclt_corr != null)? (new double [tp_tasks[0].length][][]):null;
// will use num_acc with variable number of accumulations (e.g. clusters)
//all_fpn
// used with !clt_parameters.imp.eig_use
double min_str = all_fpn ? clt_parameters.imp.min_str_fpn : clt_parameters.imp.min_str;
double min_str_sum = all_fpn ? clt_parameters.imp.min_str_sum_fpn : clt_parameters.imp.min_str_sum;
double min_str_neib = all_fpn ? clt_parameters.imp.min_str_neib_fpn : clt_parameters.imp.min_str_neib;
// used with clt_parameters.imp.eig_use
double eig_str_sum = all_fpn ? clt_parameters.imp.eig_str_sum_fpn : clt_parameters.imp.eig_str_sum;
double eig_str_neib = all_fpn ? clt_parameters.imp.eig_str_neib_fpn : clt_parameters.imp.eig_str_neib;
double corr_fz_inter = clt_parameters.getGpuFatZeroInter(first_scene.isMonochrome());
if (mb_en && (mb_vectors!=null)) { // increase fat zero when there is motion blur
corr_fz_inter *= 8;
......@@ -4131,67 +4143,152 @@ public class Interscene {
int [] corr_indices_dbg = show_2d_correlations? image_dtt.getGPU().getCorrIndices() : null;
boolean use_partial = clt_parameters.imp.use_partial;
coord_motion = image_dtt.clt_process_tl_interscene( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
// only used here to keep extra array element for disparity difference
use3D, // boolean use3D, // generate disparity difference
use_neibs, // boolean use_neibs,
fcorr_td, // final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of all selected corr pairs
null, // float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null). Can be inner null if not used in tp_tasks
null, // double [] dcorr_weight, // alternative to num_acc, compatible with CPU processing (only one non-zero enough)
clt_parameters.gpu_corr_scale, // final double gpu_corr_scale, // 0.75; // reduce GPU-generated correlation values
corr_fz_inter, // final double gpu_fat_zero, // clt_parameters.getGpuFatZero(is_mono);absolute == 30.0
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
// should it be reference or scene? Or any?
tp_tasks[0], // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMA for the integrated correlations
// to be converted to float (may be null)
dcorr_tiles, // final double [][][] dcorr_tiles, // [tile][pair_abs, sparse][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
pXpYD_ref, // ref_pXpYD, // final double [][] pXpYD, // pXpYD for the reference scene
fpn_offsets, // final double [][] fpn_offsets, // null, or per-tile X,Y offset to be blanked
fpn_radius, // final double fpn_radius, // radius to be blanked around FPN offset center
fpn_ignore_border, // final boolean fpn_ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
motion_vectors, // final double [][][] motion_vectors, // [tilesY*tilesX][][] -> [][][num_sel_sensors+1][2]
clt_parameters.imp.run_poly, // final boolean run_poly, // polynomial max, if false - centroid
use_partial, // final boolean use_partial, // find motion vectors for individual pairs, false - for sum only
clt_parameters.imp.centroid_radius,// final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
clt_parameters.imp.n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
clt_parameters.imp.td_weight, // final double td_weight, // mix correlations accumulated in TD with
clt_parameters.imp.td_neib_weight, // final double td_neib_weight, // mix correlations accumulated in TD (neibs)
clt_parameters.imp.pd_weight, // final double pd_weight, // correlations (post) accumulated in PD
clt_parameters.imp.td_nopd_only, // final boolean td_nopd_only , // only use TD accumulated data if no safe PD is available for the tile.
clt_parameters.imp.neib_notd_only, // final boolean neib_notd_only, // use neighbors only if individual TD is too weak
min_str, // final double min_str_nofpn, // = 0.25;
min_str_sum, // final double min_str_sum_nofpn, // = 0.8; // 5;
min_str_neib, // final double min_str_neib_nofpn,
clt_parameters.imp.min_str_fpn, // final double min_str, // = 0.25;
clt_parameters.imp.min_str_sum_fpn,// final double min_str_sum, // = 0.8; // 5;
clt_parameters.imp.min_str_neib_fpn,//final double min_str_neib_fpn,
clt_parameters.imp.min_neibs, // final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
clt_parameters.imp.weight_zero_neibs, // final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
half_disparity, // final double half_disparity, // 5.0; // Reduce weight twice for this disparity
clt_parameters.imp.half_avg_diff, // final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice
neibs_nofpn_only, // final boolean neibs_nofpn_only, // consolidate neighbors fot non-fpn tiles only!
redo_both, //final boolean redo_both, // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
min_num_neibs, //final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
scale_neibs_pd, //final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
scale_neibs_td, //final double scale_neibs_td, // scale threshold for the transform-domain average maximums
scale_avg_weight, //final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones
clt_parameters.tileX, // final int debug_tileX,
clt_parameters.tileY, // final int debug_tileY,
THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
debug_level);
// optional coord_motion[1][3..4] is reserved for disparity difference and strength
// final int globalDebugLevel);
if (coord_motion == null) {
System.out.println("clt_process_tl_interscene() returned null");
if (fail_reason != null) {
fail_reason[0]=FAIL_REASON_INTERSCENE;
boolean use_eigen = clt_parameters.imp.eig_use;
double [][] eigen = use_eigen? (new double[tilesX*tilesY][]) : null;
boolean eigen_debug = use_eigen && show_2d_correlations;
if (use_eigen) {
double eigen_min_abs= clt_parameters.imp.eig_min_abs ; //eigen_min_abs = 0.05; // 0.05 values below this do not count for covariance
double eigen_min_rel = clt_parameters.imp.eig_min_rel ; //0.2; // 0.2 values less than this fraction of tile's max do not count for covariance
coord_motion = image_dtt.clt_process_tl_interscene( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
// only used here to keep extra array element for disparity difference
use3D, // boolean use3D, // generate disparity difference
use_neibs, // boolean use_neibs,
fcorr_td, // final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of all selected corr pairs
null, // float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null). Can be inner null if not used in tp_tasks
null, // double [] dcorr_weight, // alternative to num_acc, compatible with CPU processing (only one non-zero enough)
clt_parameters.gpu_corr_scale, // final double gpu_corr_scale, // 0.75; // reduce GPU-generated correlation values
corr_fz_inter, // final double gpu_fat_zero, // clt_parameters.getGpuFatZero(is_mono);absolute == 30.0
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
// should it be reference or scene? Or any?
tp_tasks[0], // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMA for the integrated correlations
// to be converted to float (may be null)
dcorr_tiles, // final double [][][] dcorr_tiles, // [tile][pair_abs, sparse][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
pXpYD_ref, // ref_pXpYD, // final double [][] pXpYD, // pXpYD for the reference scene
fpn_offsets, // final double [][] fpn_offsets, // null, or per-tile X,Y offset to be blanked
fpn_radius, // final double fpn_radius, // radius to be blanked around FPN offset center
fpn_ignore_border, // final boolean fpn_ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
// motion_vectors, // final double [][][] motion_vectors, // [tilesY*tilesX][][] -> [][][num_sel_sensors+1][2]
clt_parameters.imp.run_poly, // final boolean run_poly, // polynomial max, if false - centroid
use_partial, // final boolean use_partial, // find motion vectors for individual pairs, false - for sum only
clt_parameters.imp.centroid_radius,// final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
clt_parameters.imp.n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
clt_parameters.imp.td_weight, // final double td_weight, // mix correlations accumulated in TD with
clt_parameters.imp.td_neib_weight, // final double td_neib_weight, // mix correlations accumulated in TD (neibs)
clt_parameters.imp.pd_weight, // final double pd_weight, // correlations (post) accumulated in PD
clt_parameters.imp.td_nopd_only, // final boolean td_nopd_only , // only use TD accumulated data if no safe PD is available for the tile.
clt_parameters.imp.eig_use_neibs, // final boolean eig_use_neibs, // use correlation from 9 tiles with neibs, if single-tile fails
clt_parameters.imp.eig_remove_neibs,//final boolean eig_remove_neibs, //remove weak (by-neibs) tiles if they have strong (by-single) neighbor
// min_str, // final double min_str_nofpn, // = 0.25;
eig_str_sum, // final double eig_str_sum_nofpn, // = 0.8; // 5;
eig_str_neib, // final double eig_str_neib_nofpn,
// clt_parameters.imp.min_str_fpn, // final double min_str, // = 0.25;
clt_parameters.imp.eig_str_sum_fpn,// final double eig_str_sum_fpn, // = 0.8; // 5;
clt_parameters.imp.eig_str_neib_fpn,//final double eig_str_neib_fpn,
clt_parameters.imp.min_neibs, // final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
clt_parameters.imp.weight_zero_neibs, // final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
half_disparity, // final double half_disparity, // 5.0; // Reduce weight twice for this disparity
clt_parameters.imp.half_avg_diff, // final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice
neibs_nofpn_only, // final boolean neibs_nofpn_only, // consolidate neighbors fot non-fpn tiles only!
redo_both, // final boolean redo_both, // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
min_num_neibs, // final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
scale_neibs_pd, // final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
scale_neibs_td, // final double scale_neibs_td, // scale threshold for the transform-domain average maximums
scale_avg_weight, // final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones
eigen_min_abs, // final double eigen_min_abs, // 0.05 values below this do not count for covariance
eigen_min_rel, // final double eigen_min_rel, // 0.2 values less than this fraction of tile's max do not count for covariance
clt_parameters.imp.eig_sub_frac, //final double eig_sub_frac, // 1.0; // subtract fraction of threshold {eig_min_abs,eig_min_rel} after selecting by them (0 - select only, will have pedestal)
eigen, // null, // eigen, // final double [][] eigen, // null or [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
eigen_debug, // final double [][] eigen_debug, // null or [tilesX*tilesY][]
clt_parameters.tileX, // final int debug_tileX,
clt_parameters.tileY, // final int debug_tileY,
THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
debug_level);
if (coord_motion == null) {
System.out.println("clt_process_tl_interscene() returned null");
if (fail_reason != null) {
fail_reason[0]=FAIL_REASON_INTERSCENE;
}
return null;
}
return null;
// eigen is now filled inside clt_process_tl_interscene(), and coord_motion[] is in old format if eigen[][] was supplied.
/*
if (eigen != null) { // temporarily make as old: {dx,dy,str,0,0}
int coord_motion_slice = coord_motion.length - 1; // used for motionvector - last index
for (int tile = 0; tile < coord_motion[coord_motion_slice].length; tile++ ) if ( coord_motion[coord_motion_slice][tile] != null){
eigen[tile] = new double[4];
double [] cm_old = new double[5];
System.arraycopy(coord_motion[coord_motion_slice][tile], 3, eigen[tile], 0, 4);
System.arraycopy(coord_motion[coord_motion_slice][tile], 0, cm_old, 0, 3);
coord_motion[coord_motion_slice][tile] = cm_old;
}
}
*/
} else { // if (use_eigen) { old variant
coord_motion = image_dtt.clt_process_tl_interscene_noeigen( // convert to pixel domain and process correlations already prepared in fcorr_td and/or fcorr_combo_td
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
// only used here to keep extra array element for disparity difference
use3D, // boolean use3D, // generate disparity difference
use_neibs, // boolean use_neibs,
fcorr_td, // final float [][][][] fcorr_td, // [tilesY][tilesX][pair][4*64] transform domain representation of all selected corr pairs
null, // float [][][] num_acc, // number of accumulated tiles [tilesY][tilesX][pair] (or null). Can be inner null if not used in tp_tasks
null, // double [] dcorr_weight, // alternative to num_acc, compatible with CPU processing (only one non-zero enough)
clt_parameters.gpu_corr_scale, // final double gpu_corr_scale, // 0.75; // reduce GPU-generated correlation values
corr_fz_inter, // final double gpu_fat_zero, // clt_parameters.getGpuFatZero(is_mono);absolute == 30.0
image_dtt.transform_size - 1, // final int gpu_corr_rad, // = transform_size - 1 ?
// The tp_tasks data should be decoded from GPU to get coordinates
// should it be reference or scene? Or any?
tp_tasks[0], // final TpTask [] tp_tasks, // data from the reference frame - will be applied to LMA for the integrated correlations
// to be converted to float (may be null)
dcorr_tiles, // final double [][][] dcorr_tiles, // [tile][pair_abs, sparse][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
pXpYD_ref, // ref_pXpYD, // final double [][] pXpYD, // pXpYD for the reference scene
fpn_offsets, // final double [][] fpn_offsets, // null, or per-tile X,Y offset to be blanked
fpn_radius, // final double fpn_radius, // radius to be blanked around FPN offset center
fpn_ignore_border, // final boolean fpn_ignore_border, // only if fpn_mask != null - ignore tile if maximum touches fpn_mask
motion_vectors, // final double [][][] motion_vectors, // [tilesY*tilesX][][] -> [][][num_sel_sensors+1][2]
clt_parameters.imp.run_poly, // final boolean run_poly, // polynomial max, if false - centroid
use_partial, // final boolean use_partial, // find motion vectors for individual pairs, false - for sum only
clt_parameters.imp.centroid_radius,// final double centroid_radius, // 0 - use all tile, >0 - cosine window around local max
clt_parameters.imp.n_recenter, // final int n_recenter, // when cosine window, re-center window this many times
clt_parameters.imp.td_weight, // final double td_weight, // mix correlations accumulated in TD with
clt_parameters.imp.td_neib_weight, // final double td_neib_weight, // mix correlations accumulated in TD (neibs)
clt_parameters.imp.pd_weight, // final double pd_weight, // correlations (post) accumulated in PD
clt_parameters.imp.td_nopd_only, // final boolean td_nopd_only , // only use TD accumulated data if no safe PD is available for the tile.
clt_parameters.imp.neib_notd_only, // final boolean neib_notd_only, // use neighbors only if individual TD is too weak
min_str, // final double min_str_nofpn, // = 0.25;
min_str_sum, // final double min_str_sum_nofpn, // = 0.8; // 5;
min_str_neib, // final double min_str_neib_nofpn,
clt_parameters.imp.min_str_fpn, // final double min_str, // = 0.25;
clt_parameters.imp.min_str_sum_fpn,// final double min_str_sum, // = 0.8; // 5;
clt_parameters.imp.min_str_neib_fpn,//final double min_str_neib_fpn,
clt_parameters.imp.min_neibs, // final int min_neibs, // 2; // minimal number of strong neighbors (> min_str)
clt_parameters.imp.weight_zero_neibs, // final double weight_zero_neibs,// 0.2; // Reduce weight for no-neib (1.0 for all 8)
half_disparity, // final double half_disparity, // 5.0; // Reduce weight twice for this disparity
clt_parameters.imp.half_avg_diff, // final double half_avg_diff, // 0.2; // when L2 of x,y difference from average of neibs - reduce twice
neibs_nofpn_only, // final boolean neibs_nofpn_only, // consolidate neighbors fot non-fpn tiles only!
redo_both, // final boolean redo_both, // use average of neighbors for both pd,td if any of the center tile tests (td, pd) fails
min_num_neibs, // final int min_num_neibs, // plus center, total number >= (min_num_neibs+1)
scale_neibs_pd, // final double scale_neibs_pd, // scale threshold for the pixel-domain average maximums
scale_neibs_td, // final double scale_neibs_td, // scale threshold for the transform-domain average maximums
scale_avg_weight, // final double scale_avg_weight, // reduce influence of the averaged correlations compared to the single-tile ones
clt_parameters.tileX, // final int debug_tileX,
clt_parameters.tileY, // final int debug_tileY,
THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
debug_level);
if (coord_motion == null) {
System.out.println("clt_process_tl_interscene_noigen() returned null");
if (fail_reason != null) {
fail_reason[0]=FAIL_REASON_INTERSCENE;
}
return null;
}
}
// optional coord_motion[1][3..4] is reserved for disparity difference and strength
// final int globalDebugLevel);
if (use3D) {//(scene_disparity_strength != null)
// combine motion vector with disparity_diff/strength
int num_slices = scene_disparity_strength.length;
......@@ -4202,7 +4299,7 @@ public class Interscene {
}
}
}
if (eq_en) {
if (eq_en) { // false
// double eq_weight_add = (min_str * clt_parameters.imp.pd_weight + min_str_sum * clt_parameters.imp.td_weight) /
// (clt_parameters.imp.pd_weight + clt_parameters.imp.td_weight);
double [] strength_backup = null;
......@@ -4368,10 +4465,12 @@ public class Interscene {
}
if (show_coord_motion) {
//coord_motion
String [] mvTitles = {"dx", "dy", "conf","disp_diff", "disp_str", "pX", "pY","Disp","defined"}; // ,"blurX","blurY", "blur"};
String [] mvTitles = (eigen != null)?
( new String[]{"dx", "dy", "conf","disp_diff", "disp_str", "pX", "pY","Disp","defined","eig-X", "eig-Y", "eig0", "eig1"}):
(new String []{"dx", "dy", "conf","disp_diff", "disp_str", "pX", "pY","Disp","defined"}); // ,"blurX","blurY", "blur"};
double [][] dbg_img = new double [mvTitles.length][];
for (int l = 0; l < dbg_img.length; l++) {
if (use3D || (l < 3) || (l > 4) ) {
if ((l < 9) ? (use3D || (l < 3) || (l > 4)) : (eigen != null)) {
dbg_img[l] = new double [tilesX*tilesY]; // keep unused null
Arrays.fill(dbg_img[l], Double.NaN);
}
......@@ -4388,6 +4487,16 @@ public class Interscene {
}
}
dbg_img[8][nTile] = ((coord_motion[0][nTile] != null)?1:0)+((coord_motion[0][nTile] != null)?2:0);
if ((eigen != null) && (eigen[nTile] != null)) {
dbg_img[9 + 0][nTile] = eigen[nTile][0];
dbg_img[9 + 1][nTile] = eigen[nTile][1];
double k0 = 1.0/clt_parameters.imp.eig_max_sqrt;
double rmin = clt_parameters.imp.eig_min_sqrt;
for (int i = 0; i < 2; i++) {
double k = Math.max(0, 1/Math.max(rmin, Math.sqrt(eigen[nTile][2+i]))-k0);
dbg_img[9 + 2 + i][nTile] = k;
}
}
}
ShowDoubleFloatArrays.showArrays( // out of boundary 15
dbg_img,
......@@ -4415,6 +4524,9 @@ public class Interscene {
if (fail_reason != null) {
fail_reason[0]= 0;
}
if (eigen != null) {
return new double [][][] {coord_motion[0],coord_motion[1],eigen};
}
return coord_motion; // here non-null
}
......@@ -4864,9 +4976,9 @@ public class Interscene {
}
/**
* Equalize weights of the motion vectors to boost that of important buy weak one.
* Equalize weights of the motion vectors to boost that of important by weak one.
* Process overlapping (by half, using shifted cosine weight function) supertiles
* independently and if it qualifies, increase its tiles weights equlaizing (with
* independently and if it qualifies, increase its tiles weights equalizing (with
* certain limitations) total supertile weights.
* @param coord_motion [2][tilesX*tilesY][3] input/output arrays.[0][tile][] is
* pXpYD triplet (
......
......@@ -67,6 +67,8 @@ public class IntersceneLma {
private boolean thread_invariant = true; // Do not use DoubleAdder, provide results not dependent on threads
private int num_components = 2; // 2 for 2D matching only,3 - include disparity
private double disparity_weight = 1.0; // relative weight of disparity errors
private double [][][] eig_trans = null;
public IntersceneLma(
boolean thread_invariant,
......@@ -250,13 +252,23 @@ public class IntersceneLma {
final QuadCLT reference_QuadClt,
final boolean[] param_select,
final double [] param_regweights,
final double eig_max_sqrt, // 6; // for sqrt(lambda) - consider infinity (infinite linear feature, a line)
final double eig_min_sqrt, // 1; // for sqrt(lambda) - consider infinity (infinite linear feature, a line)
//if (eigen != null) normalize by eigenvalues, recalc derivatives to eigenvectors directions
final double [][] eigen, // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
// now includes optional Disparity as the last element (for num_components==3)
final double [][] vector_XYSDS,// optical flow X,Y, confidence obtained from the correlate2DIterate()
final double [][] centers, // macrotile centers (in pixels and average disparities
boolean first_run,
final int debug_level) {
// befolre getFxDerivs
eig_trans = setEigenTransform(
eig_max_sqrt, // final double eig_max_sqrt,
eig_min_sqrt, // final double eig_min_sqrt,
eigen); // final double [][] eigen); // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
scenesCLT = new QuadCLT [] {reference_QuadClt, scene_QuadClt};
par_mask = param_select;
macrotile_centers = centers;
......@@ -339,7 +351,6 @@ public class IntersceneLma {
}
normalizeWeights(); // make full weight == 1.0; pure_weight <= 1.0;
// remeasure fx - now with regularization terms.
if (debug_level > 1) {
System.out.println("prepareLMA() 2");
}
......@@ -358,12 +369,11 @@ public class IntersceneLma {
if (num_components > 2) {
y_vector[num_components * i + 2] += vector_XYSDS[i][2];
}
}
}
if (parameters_pull != null){
for (int i = 0; i < par_indices.length; i++) {
// y_vector [i + num_components * macrotile_centers.length] += parameters_pull[i]; // - parameters_initial[i]; // scale will be combined with weights
y_vector [i + num_components * macrotile_centers.length] = parameters_pull[i]; // - parameters_initial[i]; // scale will be combined with weights
}
}
......@@ -684,7 +694,7 @@ public class IntersceneLma {
Matrix mdelta = jtjl_inv.times(jty);
if (debug_level>2) {
System.out.println("mdelta");
mdelta.print(18, 6);
mdelta.print(18, 10);
}
double scale = 1.0;
......@@ -759,7 +769,37 @@ public class IntersceneLma {
return rslt;
}
private static double [][][] setEigenTransform(
final double eig_max_sqrt,
final double eig_min_sqrt,
final double [][] eigen){ // [tilesX*tilesY]{lamb0_x,lamb0_y, lamb0, lamb1} eigenvector0[x,y],lam0,lam1
final double [][][] transform = new double[eigen.length][2][2];
final Thread[] threads = ImageDtt.newThreadArray(QuadCLT.THREADS_MAX);
final AtomicInteger ai = new AtomicInteger(0);
final double k0 = 1.0/eig_max_sqrt;
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int iTile = ai.getAndIncrement(); iTile < eigen.length; iTile = ai.getAndIncrement()) if (eigen[iTile] != null){
double [][] am = {{eigen[iTile][0],eigen[iTile][1]},{eigen[iTile][1],-eigen[iTile][0]}};
for (int i = 0; i < 2; i++) {
double k = Math.max(0, 1/Math.max(eig_min_sqrt, Math.sqrt(eigen[iTile][2+i]))-k0);
for (int j = 0; j < 2; j++) {
transform[iTile][i][j] = k* am[i][j];
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return transform;
}
private void setSamplesWeights(
final double [][] vector_XYSDS) // not regularized yet
......@@ -890,6 +930,9 @@ public class IntersceneLma {
return full_vector;
}
// if eig_trans != null, transform derivatives, keep Fx
private double [] getFxDerivs(
double [] vector,
final double [][] jt, // should be null or initialized with [vector.length][]
......@@ -974,8 +1017,18 @@ public class IntersceneLma {
if (jt != null) {
for (int i = 0; i < par_indices.length; i++) {
int indx = par_indices[i] + 1;
jt[i][num_components * iMTile + 0] = deriv_params[indx][0]; // pX
jt[i][num_components * iMTile + 1] = deriv_params[indx][1]; // pY (disparity is not used)
if (eig_trans != null) {
//********************************
// apply eig_trans here:
for (int j = 0; j < 2; j++) {
jt[i][num_components * iMTile + j] =
eig_trans[iMTile][j][0] * deriv_params[indx][0] +
eig_trans[iMTile][j][1] * deriv_params[indx][1];
}
} else {
jt[i][num_components * iMTile + 0] = deriv_params[indx][0]; // pX
jt[i][num_components * iMTile + 1] = deriv_params[indx][1]; // pY (disparity is not used)
}
if (num_components > 2) {
jt[i][num_components * iMTile + 2] = deriv_params[indx][2]; // pY (disparity is used)
}
......@@ -1053,12 +1106,18 @@ public class IntersceneLma {
final double [] fx,
final double [] rms_fp // null or [2]
) {
double [] ymfxw;
if (thread_invariant) {
return getYminusFxWeightedInvariant(fx,rms_fp); // null or [2]
ymfxw = getYminusFxWeightedInvariant(fx,rms_fp); // null or [2]
} else {
return getYminusFxWeightedFast (fx,rms_fp); // null or [2]
ymfxw = getYminusFxWeightedFast (fx,rms_fp); // null or [2]
}
}
return ymfxw;
}
// TODO: modify these 2 methods to iterate through pairs, and transform pair if (eig_trans != null)
private double [] getYminusFxWeightedInvariant(
final double [] fx,
final double [] rms_fp // null or [2]
......@@ -1068,24 +1127,68 @@ public class IntersceneLma {
final double [] wymfw = new double [fx.length];
double s_rms;
final double [] l2_arr = new double [num_samples];
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int i = ai.getAndIncrement(); i < num_samples; i = ai.getAndIncrement()) if (weights[i] > 0) {
double d = y_vector[i] - fx[i];
double wd = d * weights[i];
if (Double.isNaN(wd)) {
System.out.println("getYminusFxWeighted(): weights["+i+"]="+weights[i]+", wd="+wd+
", y_vector[i]="+y_vector[i]+", fx[i]="+fx[i]);
if (eig_trans != null) {
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nTile = ai.getAndIncrement(); nTile < macrotile_centers.length; nTile = ai.getAndIncrement()) {
int ix = num_components * nTile;
if (weights[ix] > 0) {
int iy = ix+1;
double dx = y_vector[ix] - fx[ix];
double dy = y_vector[iy] - fx[iy];
double d0 = eig_trans[nTile][0][0] * dx + eig_trans[nTile][0][1] * dy;
double d1 = eig_trans[nTile][1][0] * dx + eig_trans[nTile][1][1] * dy;
double wd0 = d0 * weights[ix];
double wd1 = d1 * weights[ix];
if (Double.isNaN(wd0) || Double.isNaN(wd1)) {
System.out.println("getYminusFxWeighted(): weights["+ix+"]="+weights[ix]+", wd0="+wd0+
", y_vector[ix]="+y_vector[ix]+", fx[ix]="+fx[ix]);
System.out.println("getYminusFxWeighted(): weights["+iy+"]="+weights[iy]+", wd1="+wd1+
", y_vector[iy]="+y_vector[iy]+", fx[iy]="+fx[iy]);
}
l2_arr[ix] = d0 * wd0;
wymfw[ix] = wd0;
l2_arr[iy] = d1 * wd1;
wymfw[iy] = wd1;
for (int j = 2; j < num_components; j++) {
int i = ix+j;
double d = y_vector[i] - fx[i];
double wd = d * weights[i];
if (Double.isNaN(wd)) {
System.out.println("getYminusFxWeighted(): weights["+i+"]="+weights[i]+", wd="+wd+
", y_vector[i]="+y_vector[i]+", fx[i]="+fx[i]);
}
//double l2 = d * wd;
l2_arr[i] = d * wd;
wymfw[i] = wd;
}
}
}
//double l2 = d * wd;
l2_arr[i] = d * wd;
wymfw[i] = wd;
}
}
};
}
ImageDtt.startAndJoin(threads);
};
}
ImageDtt.startAndJoin(threads);
} else {
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int i = ai.getAndIncrement(); i < num_samples; i = ai.getAndIncrement()) if (weights[i] > 0) {
double d = y_vector[i] - fx[i];
double wd = d * weights[i];
if (Double.isNaN(wd)) {
System.out.println("getYminusFxWeighted(): weights["+i+"]="+weights[i]+", wd="+wd+
", y_vector[i]="+y_vector[i]+", fx[i]="+fx[i]);
}
//double l2 = d * wd;
l2_arr[i] = d * wd;
wymfw[i] = wd;
}
}
};
}
ImageDtt.startAndJoin(threads);
}
s_rms = 0.0;
for (double l2:l2_arr) {
s_rms += l2;
......@@ -1108,7 +1211,7 @@ public class IntersceneLma {
private double [] getYminusFxWeightedFast(
private double [] getYminusFxWeightedFast( // problems. at least with eigen?
final double [] fx,
final double [] rms_fp // null or [2]
) {
......@@ -1117,25 +1220,69 @@ public class IntersceneLma {
final double [] wymfw = new double [fx.length];
final AtomicInteger ati = new AtomicInteger(0);
final double [] l2_arr = new double [threads.length];
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
int thread_num = ati.getAndIncrement();
for (int i = ai.getAndIncrement(); i < num_samples; i = ai.getAndIncrement()) if (weights[i] > 0) {
double d = y_vector[i] - fx[i];
double wd = d * weights[i];
if (Double.isNaN(wd)) {
System.out.println("getYminusFxWeighted(): weights["+i+"]="+weights[i]+", wd="+wd+
", y_vector[i]="+y_vector[i]+", fx[i]="+fx[i]);
if (eig_trans != null) {
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
int thread_num = ati.getAndIncrement();
for (int nTile = ai.getAndIncrement(); nTile < macrotile_centers.length; nTile = ai.getAndIncrement()) {
int ix = num_components * nTile;
if (weights[ix] > 0) {
int iy = ix+1;
double dx = y_vector[ix] - fx[ix];
double dy = y_vector[iy] - fx[iy];
double d0 = eig_trans[nTile][0][0] * dx + eig_trans[nTile][0][1] * dy;
double d1 = eig_trans[nTile][1][0] * dx + eig_trans[nTile][1][1] * dy;
double wd0 = d0 * weights[ix];
double wd1 = d1 * weights[ix];
if (Double.isNaN(wd0) || Double.isNaN(wd1)) {
System.out.println("getYminusFxWeighted(): weights["+ix+"]="+weights[ix]+", wd0="+wd0+
", y_vector[ix]="+y_vector[ix]+", fx[ix]="+fx[ix]);
System.out.println("getYminusFxWeighted(): weights["+iy+"]="+weights[iy]+", wd1="+wd1+
", y_vector[iy]="+y_vector[iy]+", fx[iy]="+fx[iy]);
}
l2_arr[thread_num] += d0 * wd0;
wymfw[ix] = wd0;
l2_arr[thread_num] += d1 * wd1;
wymfw[iy] = wd1;
for (int j = 2; j < num_components; j++) {
int i = ix+j;
double d = y_vector[i] - fx[i];
double wd = d * weights[i];
if (Double.isNaN(wd)) {
System.out.println("getYminusFxWeighted(): weights["+i+"]="+weights[i]+", wd="+wd+
", y_vector[i]="+y_vector[i]+", fx[i]="+fx[i]);
}
//double l2 = d * wd;
l2_arr[thread_num] += d * wd;
wymfw[i] = wd;
}
}
}
//double l2 = d * wd;
l2_arr[thread_num] += d * wd;
wymfw[i] = wd;
}
}
};
}
ImageDtt.startAndJoin(threads);
};
}
ImageDtt.startAndJoin(threads);
} else {
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
int thread_num = ati.getAndIncrement();
for (int i = ai.getAndIncrement(); i < num_samples; i = ai.getAndIncrement()) if (weights[i] > 0) {
double d = y_vector[i] - fx[i];
double wd = d * weights[i];
if (Double.isNaN(wd)) {
System.out.println("getYminusFxWeighted(): weights["+i+"]="+weights[i]+", wd="+wd+
", y_vector[i]="+y_vector[i]+", fx[i]="+fx[i]);
}
l2_arr[thread_num] += d * wd;
wymfw[i] = wd;
}
}
};
}
ImageDtt.startAndJoin(threads);
}
double s_rms = 0.0;
for (double l2:l2_arr) {
s_rms += l2;
......@@ -1148,7 +1295,7 @@ public class IntersceneLma {
s_rms += d * wd;
wymfw[indx] = wd;
}
double rms = Math.sqrt(s_rms); // assuming sum_weights == 1.0; /pure_weight); shey should be re-normalized after adding regularization
double rms = Math.sqrt(s_rms); // assuming sum_weights == 1.0; /pure_weight); they should be re-normalized after adding regularization
if (rms_fp != null) {
rms_fp[0] = rms;
rms_fp[1] = rms_pure;
......
......@@ -437,6 +437,31 @@ public class IntersceneMatchParameters {
public double half_disparity = 5.0; // Reduce weight twice for this disparity
public double half_avg_diff = 0.2; // when L2 of x,y difference from average of neibs - reduce twice
// new method with alignment to eigenvectors. Keep above for comparison, and possibly for front-looking
public boolean eig_use = false; // use eigenvectors alignment, false for old settings
public double eig_str_sum = 0.22; // 0.2; // 0.33; // tiles w/o FPN: minimal correlation strength for TD-accumulated layer
public double eig_str_sum_fpn = 0.33; // 0.5; // 0.8; // minimal correlation strength for TD-accumulated layer
public double eig_str_neib = 0.1; // 0.33; // tiles w/o FPN: minimal correlation strength for TD-accumulated layer
public double eig_str_neib_fpn = 0.15; // 0.8; // minimal correlation strength for TD-accumulated neighbors layer
public double eig_min_abs = 0.05; // minimal absolute corr value to keep for centroid and eigen filtering of connected
public double eig_min_rel = 0.25; // minimal relative to max corr value to keep for centroid and eigen filtering of connected
public double eig_sub_frac = 1.0; // subtract fraction of threshold {eig_min_abs,eig_min_rel} after selecting by them (0 - select only, will have pedestal)
public double eig_max_sqrt = 6.0; // for sqrt(lambda) - consider infinity (infinite linear feature, a line)
public double eig_min_sqrt = 1.0; // for sqrt(lambda) - limit minimal sqrt(lambda) - can be sharp for very small max
public boolean eig_use_neibs = true; // use correlation from 9 tiles with neibs, if single-tile fails
public boolean eig_remove_neibs = true; // remove weak (by-neibs) tiles if they have strong (by-single) neighbor
/*
min_str_sum_nofpn 0.22
min_str_neib_nofpn 0.25
min_str_fpn 0.2
min_str_sum_fpn 0.22
min_str_neib_fpn 0.35
*/
// Detect initial match
public boolean use_combo_reliable = true; // use combo dsi if available for relaible tiles
public boolean ref_need_lma = true; // need LMA output for reliable tiles (no combo available)
......@@ -1307,6 +1332,36 @@ public class IntersceneMatchParameters {
gd.addNumericField("Difference from neighbors average ", this.half_avg_diff, 5,7,"",
"Reduce twice for high difference from neighbors average.");
gd.addMessage ("Align to eigenvectors, replaces above group");
gd.addCheckbox ("Use eigenvectors", this.eig_use,
"Use adjustment based on aligned/normalized offset vectors, errors sqrt(eigenvalue) normalized");
gd.addNumericField("Minimal correlation strength (intrascene)",this.eig_str_sum, 5,7,"",
"Minimal correlation strength for transform-domain averaging among sensors. Weeker tiles results are removed.");
gd.addNumericField("Minimal correlation strength (intrascene) w/FPN",this.eig_str_sum_fpn, 5,7,"",
"Similar to above, but for small offsets where FPN correlation may be present");
gd.addNumericField("Minimal correlation neighbors strength", this.eig_str_neib, 5,7,"",
"Minimal correlation strength for transform-domain averaging among sensors and neighbors. Weeker tiles results are removed.");
gd.addNumericField("Minimal correlation neighbors strength w/FPN", this.eig_str_neib_fpn, 5,7,"",
"Similar to above, but for small offsets where FPN correlation may be present");
gd.addNumericField("Minimal correlation absolute", this.eig_min_abs, 5,7,"",
"Minimal absolute corr value to keep for centroid and eigen filtering of connected.");
gd.addNumericField("Minimal correlation relative", this.eig_min_rel, 5,7,"",
"Minimal relative to max corr value to keep for centroid and eigen filtering of connected.");
gd.addNumericField("Pedestal supress", this.eig_sub_frac, 5,7,"",
"1.0 - subract threshold value before centroid and covariance, 0.0 - keep original correlation values.");
gd.addNumericField("Infinity radius", this.eig_max_sqrt, 5,7,"",
"For sqrt(egenvalue)==radius - consider infinity (infinite linear feature, a line).");
gd.addNumericField("Minimal radius", this.eig_min_sqrt, 5,7,"",
"For sqrt(egenvalue)==radius - do not trust too sharp (caused by small spot over threshold).");
gd.addCheckbox ("Use neighbors if single fails", this.eig_use_neibs,
"Use correlation from 9 tiles with neibs, if single-tile fails");
gd.addCheckbox ("Remove weak by strong neighbors", this.eig_remove_neibs,
"Remove weak (by-neibs) tiles if they have strong (by-single) neighbor");
gd.addMessage ("Filtering tiles for interscene matching");
gd.addCheckbox ("Use combo DSI (if available) for reliable", this.use_combo_reliable,
......@@ -1980,6 +2035,20 @@ public class IntersceneMatchParameters {
this.weight_zero_neibs = gd.getNextNumber();
this.half_disparity = gd.getNextNumber();
this.half_avg_diff = gd.getNextNumber();
this.eig_use = gd.getNextBoolean();
this.eig_str_sum = gd.getNextNumber();
this.eig_str_sum_fpn = gd.getNextNumber();
this.eig_str_neib = gd.getNextNumber();
this.eig_str_neib_fpn = gd.getNextNumber();
this.eig_min_abs = gd.getNextNumber();
this.eig_min_rel = gd.getNextNumber();
this.eig_sub_frac = gd.getNextNumber();
this.eig_max_sqrt = gd.getNextNumber();
this.eig_min_sqrt = gd.getNextNumber();
this.eig_use_neibs = gd.getNextBoolean();
this.eig_remove_neibs = gd.getNextBoolean();
this.use_combo_reliable = gd.getNextBoolean();
this.ref_need_lma = gd.getNextBoolean();
this.ref_need_lma_combo = gd.getNextBoolean();
......@@ -2522,6 +2591,19 @@ public class IntersceneMatchParameters {
properties.setProperty(prefix+"half_disparity", this.half_disparity+""); // double
properties.setProperty(prefix+"half_avg_diff", this.half_avg_diff+""); // double
properties.setProperty(prefix+"eig_use", this.eig_use+""); // boolean
properties.setProperty(prefix+"eig_str_sum", this.eig_str_sum+""); // double
properties.setProperty(prefix+"eig_str_sum_fpn", this.eig_str_sum_fpn+""); // double
properties.setProperty(prefix+"eig_str_neib", this.eig_str_neib+""); // double
properties.setProperty(prefix+"eig_str_neib_fpn", this.eig_str_neib_fpn+""); // double
properties.setProperty(prefix+"eig_min_abs", this.eig_min_abs+""); // double
properties.setProperty(prefix+"eig_min_rel", this.eig_min_rel+""); // double
properties.setProperty(prefix+"eig_sub_frac", this.eig_sub_frac+""); // double
properties.setProperty(prefix+"eig_max_sqrt", this.eig_max_sqrt+""); // double
properties.setProperty(prefix+"eig_min_sqrt", this.eig_min_sqrt+""); // double
properties.setProperty(prefix+"eig_use_neibs", this.eig_use_neibs+""); // boolean
properties.setProperty(prefix+"eig_remove_neibs", this.eig_remove_neibs+""); // boolean
properties.setProperty(prefix+"use_combo_reliable", this.use_combo_reliable+""); // boolean
properties.setProperty(prefix+"ref_need_lma", this.ref_need_lma+""); // boolean
properties.setProperty(prefix+"ref_need_lma_combo", this.ref_need_lma_combo+""); // boolean
......@@ -3026,6 +3108,19 @@ public class IntersceneMatchParameters {
if (properties.getProperty(prefix+"half_disparity")!=null) this.half_disparity=Double.parseDouble(properties.getProperty(prefix+"half_disparity"));
if (properties.getProperty(prefix+"half_avg_diff")!=null) this.half_avg_diff=Double.parseDouble(properties.getProperty(prefix+"half_avg_diff"));
if (properties.getProperty(prefix+"eig_use")!=null) this.eig_use=Boolean.parseBoolean(properties.getProperty(prefix+"eig_use"));
if (properties.getProperty(prefix+"eig_str_sum")!=null) this.eig_str_sum=Double.parseDouble(properties.getProperty(prefix+"eig_str_sum"));
if (properties.getProperty(prefix+"eig_str_sum_fpn")!=null) this.eig_str_sum_fpn=Double.parseDouble(properties.getProperty(prefix+"eig_str_sum_fpn"));
if (properties.getProperty(prefix+"eig_str_neib")!=null) this.eig_str_neib=Double.parseDouble(properties.getProperty(prefix+"eig_str_neib"));
if (properties.getProperty(prefix+"eig_str_neib_fpn")!=null) this.eig_str_neib_fpn=Double.parseDouble(properties.getProperty(prefix+"eig_str_neib_fpn"));
if (properties.getProperty(prefix+"eig_min_abs")!=null) this.eig_min_abs=Double.parseDouble(properties.getProperty(prefix+"eig_min_abs"));
if (properties.getProperty(prefix+"eig_min_rel")!=null) this.eig_min_rel=Double.parseDouble(properties.getProperty(prefix+"eig_min_rel"));
if (properties.getProperty(prefix+"eig_sub_frac")!=null) this.eig_sub_frac=Double.parseDouble(properties.getProperty(prefix+"eig_sub_frac"));
if (properties.getProperty(prefix+"eig_max_sqrt")!=null) this.eig_max_sqrt=Double.parseDouble(properties.getProperty(prefix+"eig_max_sqrt"));
if (properties.getProperty(prefix+"eig_min_sqrt")!=null) this.eig_min_sqrt=Double.parseDouble(properties.getProperty(prefix+"eig_min_sqrt"));
if (properties.getProperty(prefix+"eig_use_neibs")!=null) this.eig_use_neibs=Boolean.parseBoolean(properties.getProperty(prefix+"eig_use_neibs"));
if (properties.getProperty(prefix+"eig_remove_neibs")!=null) this.eig_remove_neibs=Boolean.parseBoolean(properties.getProperty(prefix+"eig_remove_neibs"));
if (properties.getProperty(prefix+"use_combo_reliable")!=null) this.use_combo_reliable=Boolean.parseBoolean(properties.getProperty(prefix+"use_combo_reliable"));
else if (properties.getProperty(prefix+"use_combo_relaible")!=null) this.use_combo_reliable=Boolean.parseBoolean(properties.getProperty(prefix+"use_combo_relaible"));
if (properties.getProperty(prefix+"ref_need_lma")!=null) this.ref_need_lma=Boolean.parseBoolean(properties.getProperty(prefix+"ref_need_lma"));
......@@ -3533,12 +3628,24 @@ public class IntersceneMatchParameters {
imp.min_str_neib = this.min_str_neib;
imp.min_str_neib_fpn = this.min_str_neib_fpn;
imp.min_neibs = this.min_neibs;
imp.weight_zero_neibs = this.weight_zero_neibs;
imp.half_disparity = this.half_disparity;
imp.half_avg_diff = this.half_avg_diff;
imp.eig_use = this.eig_use;
imp.eig_str_sum = this.eig_str_sum;
imp.eig_str_sum_fpn = this.eig_str_sum_fpn;
imp.eig_str_neib = this.eig_str_neib;
imp.eig_str_neib_fpn = this.eig_str_neib_fpn;
imp.eig_min_abs = this.eig_min_abs;
imp.eig_min_rel = this.eig_min_rel;
imp.eig_sub_frac = this.eig_sub_frac;
imp.eig_max_sqrt = this.eig_max_sqrt;
imp.eig_min_sqrt = this.eig_min_sqrt;
imp.eig_use_neibs = this.eig_use_neibs;
imp.eig_remove_neibs = this.eig_remove_neibs;
imp.use_combo_reliable = this.use_combo_reliable;
imp.ref_need_lma = this.ref_need_lma;
imp.ref_need_lma_combo = this.ref_need_lma_combo;
......
......@@ -1900,7 +1900,7 @@ public class QuaternionLma {
Matrix mdelta = jtjl_inv.times(jty);
if (debug_level>2) {
System.out.println("mdelta");
mdelta.print(18, 6);
mdelta.print(18, 10);
}
double scale = 1.0;
......
......@@ -1268,6 +1268,58 @@ public class TileNeibs{
}
return enum_clust_ordered;
}
/**
* Get tiles, connected to the seed tile
* @param tiles selected tiles, size should be sizeX * sizeY
* @param tiles
* @param seedX
* @param seedY
* @return boolean array of tiles that are connected to the seed tile
*/
public boolean [] getConnected(
boolean [] tiles,
int seedX,
int seedY)
{
int [] waves = new int [tiles.length];
boolean [] rslt = new boolean [tiles.length];
for (int i = 0; i < tiles.length; i++) waves[i] = tiles[i] ? 0: -1;
ArrayList<Integer> front = new ArrayList<Integer>();
int [] enum_clust = new int[tiles.length];
if ((seedX < 0) || (seedY < 0) || (seedX >= sizeX) || (seedY>= sizeY) || (tiles.length != (sizeX*sizeY)) ) {
return null;
}
int start_indx = seedX + seedY * sizeX;
if (!tiles[start_indx]) {
return rslt;
}
Integer ipx = start_indx;
Integer ipx1;
front.clear();
int area = 1;
waves[ipx] = area;
rslt[ipx] = true;
front.add(ipx);
while (!front.isEmpty()) {
ipx = front.remove(0);// get oldest element
for (int d = 0; d < dirs; d++){
ipx1 = getNeibIndex(ipx, d);
if (ipx1 >= 0){
if (waves[ipx1] == 0) {
area++;
waves[ipx1] = area;
rslt[ipx1] = true;
front.add(ipx1);
}
}
}
}
return rslt;
}
public static int getMax(
int [] data)
{
......
......@@ -1697,6 +1697,28 @@ public class StructureFromMotion {
final SfmCorr [] sfmCorr = new SfmCorr [accum_PD.length];
final double [][][] data_PD = (neibs_PD == null) ? (new double[][][] {accum_PD}):(new double[][][] {accum_PD,neibs_PD});
boolean show_2d_correlations = debugLevel>10;
if (show_2d_correlations) {
String [] dbg_titles = (neibs_PD == null) ? (new String[] {"accum"}):(new String[] {"accum","neibs"});
double [][] dbg_2d_corrs = ImageDtt.corr_partial_dbg( // not used in lwir
data_PD, // final double [][][] corr_data, // [layer][tile][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate
tilesX, // final int tilesX,
corr_size, //final int corr_size, // 15
clt_parameters.corr_border_contrast, // final double border_contrast,
debugLevel); // final int globalDebugLevel)
int tilesY = data_PD[0].length/tilesX;
ShowDoubleFloatArrays.showArrays(
dbg_2d_corrs,
tilesX * (corr_size + 1),
tilesY * (corr_size + 1),
true,
"sfm_corr2d-"+scene_pairs[num_pairs-1][1].getImageName()+"-"+
scene_pairs[0][0].getImageName()+"-"+num_pairs+"_fz"+((int) corr_fz_inter),
dbg_titles);
}
final double [][][] disp_corr = new double [data_PD.length][][];
for (int i = 0; i < data_PD.length; i++) {
disp_corr[i] = getSfmDisparityCorrectionStrength(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment