Skip to content

I
imagej-elphel
Commit c216c7e4 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

Implemented CLT accumulation for virtual orientation

parent 952177ec
Hide whitespace changes
Inline Side-by-side
Showing with 810 additions and 27 deletions
src/main/java/com/elphel/imagej/gpu/GpuQuad.java
View file @ c216c7e4
......@@ -1114,6 +1114,17 @@ public class GpuQuad{ // quad camera description
copyD2H.Height = img_height; // /4;
cuMemcpy2D(copyD2H); // run copy
}
public int [] getWH(boolean use_ref) {
return use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
}
public int getNumColors() {
return num_colors;
}
public int getNumSensors() {
return num_cams;
}
public int getCltSize(boolean use_ref) { // per camera, in floats
int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
......@@ -1121,7 +1132,16 @@ public class GpuQuad{ // quad camera description
int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;
return tilesY*tilesX*num_colors* 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
}
/*
public int getCltLength(boolean use_ref) {
int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
int tilesX = wh[0] / GPUTileProcessor.DTT_SIZE;
int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;
int tile_size_td = getCltSize(use_ref); // 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
int num_tiles = tilesY*tilesX*num_colors;
return num_tiles* tile_size_td;
}
*/
public float [][] getCltData( // only for color=0
boolean use_ref){
CUdeviceptr [] gpu_sel_clt_h = use_ref ? gpu_clt_ref_h : gpu_clt_h;
......@@ -1208,24 +1228,36 @@ public class GpuQuad{ // quad camera description
}
return;
}
/*
public void setBayerImage(
float [] bayer_image,
int ncam) {
public void setCltData( // for testing only
int ncam,
float [] fclt, //
boolean use_ref){
int clt_size = getCltSize(use_ref);
if (fclt.length != clt_size) {
System.out.println("getCltData(): wrong array size: got ["+fclt.length+"], "+
"should be ["+clt_size+"]");
return;
}
CUdeviceptr [] gpu_sel_clt_h = use_ref ? gpu_clt_ref_h : gpu_clt_h;
int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
int tilesX = wh[0] / GPUTileProcessor.DTT_SIZE;
int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;
int tile_size_td = 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
int num_tiles = tilesY*tilesX*num_colors;
CUDA_MEMCPY2D copyH2D = new CUDA_MEMCPY2D();
copyH2D.srcMemoryType = CUmemorytype.CU_MEMORYTYPE_HOST;
copyH2D.srcHost = Pointer.to(bayer_image);
copyH2D.srcPitch = img_width*Sizeof.FLOAT; // width_in_bytes;
copyH2D.srcHost = Pointer.to(fclt);
copyH2D.srcPitch = tile_size_td * Sizeof.FLOAT;
copyH2D.dstMemoryType = CUmemorytype.CU_MEMORYTYPE_DEVICE;
copyH2D.dstDevice = gpu_bayer_h[ncam]; // src_dpointer;
copyH2D.dstPitch = mclt_stride *Sizeof.FLOAT; // device_stride[0];
copyH2D.WidthInBytes = img_width*Sizeof.FLOAT; // width_in_bytes;
copyH2D.Height = img_height; // /4;
cuMemcpy2D(copyH2D);
copyH2D.dstDevice = gpu_sel_clt_h[ncam];
copyH2D.dstPitch = tile_size_td * Sizeof.FLOAT;
copyH2D.WidthInBytes = tile_size_td * Sizeof.FLOAT;
copyH2D.Height = num_tiles;
cuMemcpy2D(copyH2D); // run copy
return;
}
*/
/**
......
src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java
View file @ c216c7e4
......@@ -55,6 +55,7 @@ import com.elphel.imagej.common.PolynomialApproximation;
import com.elphel.imagej.common.ShowDoubleFloatArrays;
import com.elphel.imagej.correction.CorrectionColorProc;
import com.elphel.imagej.cuas.CuasCenterLma;
import com.elphel.imagej.gpu.GPUTileProcessor;
import com.elphel.imagej.gpu.GpuQuad;
import com.elphel.imagej.gpu.TpTask;
import com.elphel.imagej.ims.Did_ins_1;
......@@ -5914,6 +5915,67 @@ public class OpticalFlow {
cuas_atr = new double [] { center_ATR[0][0], center_ATR[0][1], center_ATR[0][2]};
}
boolean combine_clt = true;
if (combine_clt) {
boolean apply_clt = true; // set GPU with data
boolean show_clt = true;
boolean merge_clt = true;
QuadCLT ref_clt = quadCLTs[ref_index];
int sensor_mask_clt = -1; // all
if (combo_dsn_final == null) {
combo_dsn_final =quadCLTs[ref_index].restoreComboDSI(true); // also sets quadCLTs[ref_index].dsi and blue sky
}
double [][] dls = {
combo_dsn_final[COMBO_DSN_INDX_DISP],
combo_dsn_final[COMBO_DSN_INDX_LMA],
combo_dsn_final[COMBO_DSN_INDX_STRENGTH]
};
double [][] ds_clt = conditionInitialDS(
true, // boolean use_conf, // use configuration parameters, false - use following
clt_parameters, // CLTParameters clt_parameters,
dls, // double [][] dls
quadCLTs[ref_index], // QuadCLT scene,
debugLevel);
float [][] combo_seq_clt = getTDComboSceneSequence(
clt_parameters, // CLTParameters clt_parameters,
merge_clt, // boolean merge_all,
sensor_mask_clt, // int sensor_mask,
null, // Rectangle fov_tiles,
ZERO3, // double [] stereo_xyz, // offset reference camera {x,y,z}
cuas_atr, // double [] stereo_atr_in, // offset reference orientation (cuas)
ds_clt[0], // double [] ref_disparity,
quadCLTs, // QuadCLT [] quadCLTs,
ref_clt, // QuadCLT refCLT, // should be the same instance if one of quadCLTs
debugLevel); // int debugLevel)
int [] whc = new int[3];
if (apply_clt) { // set GPU with data
quadCLTs[ref_index].setComboToTD(
combo_seq_clt, // final float [][] fclt,
merge_clt, // final boolean merge_channels, // duplicate same data to all selected channels
sensor_mask_clt, // final int sensor_mask, // only if merge_channels
whc, // final int [] whc, // if int[2], will return width, height
false); // final boolean use_reference);
if (show_clt) {
String suffix="-virtual";
ImagePlus imp_virtual = ref_clt.renderFromTD ( // do we need to update gpuQuad ?
sensor_mask_clt, // final int sensor_mask,
merge_clt, // boolean merge_channels,
clt_parameters, // CLTParameters clt_parameters,
clt_parameters.getColorProcParameters(ref_clt.isAux()), //ColorProcParameters colorProcParameters,
clt_parameters.getRGBParameters(), //EyesisCorrectionParameters.RGBParameters rgbParameters,\
whc, // null, // int [] wh,
false, // toRGB, // boolean toRGB,
false, // use_reference, // boolean use_reference
suffix); // String suffix)
imp_virtual.show();
}
}
}
if (generate_egomotion) {
boolean ego_show = !clt_parameters.batch_run; //true;
String ego_path = quadCLTs[ref_index].getX3dDirectory()+Prefs.getFileSeparator()+
......@@ -7859,16 +7921,6 @@ public class OpticalFlow {
double [][] ref_pXpYD;
double [][] ref_pXpYD_or_null = null; // debugging cuas mode keeping old
if (mode_cuas) { // && (dbg_scene > 0)) {
/*
double [][] ref_pXpYD_0 = transformToScenePxPyD( // now should work with offset ref_scene
fov_tiles, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // final double [] disparity_ref, // invalid tiles - NaN in disparity
ZERO3, // stereo_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // stereo_atr, // ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
quadCLTs[ref_index], // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
threadsMax); // int threadsMax)
*/
int around = 2;
double around_sigma = 4.0;
int num_virtual_refines = 2;
......@@ -8115,6 +8167,210 @@ public class OpticalFlow {
return imp_scenes;
}
public static float [][] getTDComboSceneSequence(
CLTParameters clt_parameters,
boolean merge_all,
int sensor_mask,
Rectangle fov_tiles,
double [] stereo_xyz, // offset reference camera {x,y,z}
double [] stereo_atr_in, // offset reference orientation (cuas)
double [] ref_disparity,
QuadCLT [] quadCLTs,
QuadCLT refCLT, // should be the same instance if one of quadCLTs
// int ref_index,
int debugLevel) {
double [] stereo_atr = (stereo_atr_in != null)? stereo_atr_in: ZERO3; // maybe later play with rotated camera
boolean mode_cuas = (stereo_atr[0] != 0) || (stereo_atr[1] != 0) || (stereo_atr[2] != 0);
boolean mb_en = clt_parameters.imp.mb_en && (fov_tiles==null);
double mb_tau = clt_parameters.imp.mb_tau; // 0.008; // time constant, sec
double mb_max_gain = clt_parameters.imp.mb_max_gain; // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
ErsCorrection ers_reference = refCLT.getErsCorrection();
int dbg_scene = -95;
double [][] ref_pXpYD;
double [][] ref_pXpYD_or_null = null; // debugging cuas mode keeping old
if (mode_cuas) { // && (dbg_scene > 0)) {
int around = 2;
double around_sigma = 4.0;
int num_virtual_refines = 2;
String debugSuffix= null; // "virtual";
ref_pXpYD= transformFromVirtual(
ref_disparity, // double [] disparity_ref,
stereo_xyz, // final double [] scene_xyz, // camera center in world (reference) coordinates
stereo_atr, // final double [] scene_atr, // camera orientation relative to world (reference) frame
refCLT, // quadCLTs[ref_index], // final QuadCLT reference_QuadClt,
around, // final int around, // 2 search around for interpolation
around_sigma, // final double sigma,
num_virtual_refines, // final int num_refines,
debugSuffix); // final String debugSuffix)
ref_pXpYD_or_null = ref_pXpYD;
// quadCLTs[ref_index].getErsCorrection().setupERS();
refCLT.getErsCorrection().setupERS();
System.out.println("Calculated virtual_PxPyD");
} else {
ref_pXpYD = transformToScenePxPyD( // now should work with offset ref_scene
fov_tiles, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
ref_disparity, // final double [] disparity_ref, // invalid tiles - NaN in disparity
ZERO3, // stereo_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
ZERO3, // stereo_atr, // ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
refCLT, // quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
refCLT, // quadCLTs[ref_index], // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
QuadCLT.THREADS_MAX); // int threadsMax)
}
// quadCLTs[0].getCltLength(boolean use_ref)
int sc0 = -1;
for (int nscene = 0; nscene < quadCLTs.length ; nscene++) if (quadCLTs[nscene] != null){
sc0 = nscene;
break;
}
final float [][] sumFclt = new float [merge_all? 1 : quadCLTs[sc0].getNumSensors()][quadCLTs[sc0].getCltSize(false)];
final int [][] numAcc = new int [sumFclt.length][sumFclt[0].length];
// next two to improve multithreading performance
final int tile_size_td = 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
final int tiles_td = sumFclt[0].length/tile_size_td;
final int tiles_td_all = tiles_td * sumFclt.length; // usually sumFclt.length==1
final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
final AtomicInteger ai = new AtomicInteger(0);
for (int nscene = 0; nscene < quadCLTs.length ; nscene++) if (quadCLTs[nscene] != null){
if (nscene== dbg_scene) {
System.out.println("renderSceneSequence(): nscene = "+nscene);
}
String ts = quadCLTs[nscene].getImageName();
double [] scene_xyz = ZERO3;
double [] scene_atr = ZERO3;
// if (nscene != ref_index) { // Check even for raw, so video frames will match in all modes
if (quadCLTs[nscene] != refCLT) { // Check even for raw, so video frames will match in all modes
scene_xyz = ers_reference.getSceneXYZ(ts);
scene_atr = ers_reference.getSceneATR(ts);
if ((scene_atr==null) || (scene_xyz == null)) {
continue;
}
double [] scene_ers_xyz_dt = ers_reference.getSceneErsXYZ_dt(ts);
double [] scene_ers_atr_dt = ers_reference.getSceneErsATR_dt(ts);
quadCLTs[nscene].getErsCorrection().setErsDt(
scene_ers_xyz_dt, // double [] ers_xyz_dt,
scene_ers_atr_dt); // double [] ers_atr_dt)(ers_scene_original_xyz_dt);
}
if (!mode_cuas && (stereo_xyz != null)) { // offset all, including reference scene - now always, it is never null
double [][] combo_xyzatr = ErsCorrection.combineXYZATR(
stereo_xyz, // double [] reference_xyz,
stereo_atr, // double [] reference_atr,
scene_xyz, // double [] scene_xyz,
scene_atr); // double [] scene_atr)
scene_xyz = combo_xyzatr[0];
scene_atr = combo_xyzatr[1];
}
int sm = merge_all? -1: sensor_mask;
float [][] fclt = null;
double [][] dxyzatr_dt = null;
// should get velocities from HashMap at reference scene from timestamp , not re-calculate.
if (mb_en) {
// dxyzatr_dt = getVelocities(
// quadCLTs, // QuadCLT [] quadCLTs,
// nscene); // int nscene)
dxyzatr_dt = new double[][] { // for all, including ref
quadCLTs[nscene].getErsCorrection().getErsXYZ_dt(),
quadCLTs[nscene].getErsCorrection().getErsATR_dt()};
}
if (mb_en && (dxyzatr_dt != null)) {
double [][] motion_blur = getMotionBlur(
refCLT, // quadCLTs[ref_index], // QuadCLT ref_scene,
quadCLTs[nscene], // QuadCLT scene, // can be the same as ref_scene
ref_pXpYD, // double [][] ref_pXpYD, // here it is scene, not reference!
scene_xyz, // double [] camera_xyz,
scene_atr, // double [] camera_atr,
dxyzatr_dt[0], // double [] camera_xyz_dt,
dxyzatr_dt[1], // double [] camera_atr_dt,
0, // int shrink_gaps, // will gaps, but not more that grow by this
debugLevel); // int debug_level)
fclt = QuadCLT.getTDCombo(
sm, // final int sensor_mask,
merge_all, // final boolean merge_channels,
null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
clt_parameters, // CLTParameters clt_parameters,
ref_disparity, // double [] disparity_ref,
ref_pXpYD_or_null, // double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
motion_blur, // double [][] mb_vectors, //
scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene], // final QuadCLT scene,
refCLT, // quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
true, // final boolean show_nan,
QuadCLT.THREADS_MAX, // int threadsMax,
debugLevel); // final int debugLevel)
} else {
fclt = QuadCLT.getTDCombo(
sm, // final int sensor_mask,
merge_all, // final boolean merge_channels,
null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
clt_parameters, // CLTParameters clt_parameters,
ref_disparity, // double [] disparity_ref,
ref_pXpYD_or_null, // double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
0, // double mb_tau, // 0.008; // time constant, sec
0, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
null, // double [][] mb_vectors, //
scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
quadCLTs[nscene], // final QuadCLT scene,
refCLT, // quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
true, // final boolean show_nan,
QuadCLT.THREADS_MAX, // int threadsMax,
debugLevel); // final int debugLevel)
}
final float [][] ffclt = fclt;
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nTileAll = ai.getAndIncrement(); nTileAll < tiles_td_all; nTileAll = ai.getAndIncrement()) {
int ntile = nTileAll % tiles_td;
int nsens = nTileAll / tiles_td;
int indx0 = ntile * tile_size_td;
int indx1 = indx0 + tile_size_td;
for (int indx = indx0; indx < indx1; indx++) {
float d =ffclt[nsens][indx];
if (!Float.isNaN(d)){
sumFclt[nsens][indx] += d;
numAcc[nsens][indx]++ ;
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
}
// calculate averages
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nTileAll = ai.getAndIncrement(); nTileAll < tiles_td_all; nTileAll = ai.getAndIncrement()) {
int ntile = nTileAll % tiles_td;
int nsens = nTileAll / tiles_td;
int indx0 = ntile * tile_size_td;
int indx1 = indx0 + tile_size_td;
for (int indx = indx0; indx < indx1; indx++) {
if (numAcc[nsens][indx] > 0) {
sumFclt[nsens][indx]/=numAcc[nsens][indx];
} else {
sumFclt[nsens][indx] = Float.NaN;
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return sumFclt;
}
public static double [][] getSceneSZXY(
......
src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java
View file @ c216c7e4
......@@ -1311,7 +1311,7 @@ public class QuadCLT extends QuadCLTCPU {
* @return
*/
public static ImagePlus renderGPUFromDSI(
public static ImagePlus renderGPUFromDSI_old( // remove after testing the replacement method
final int sensor_mask,
final boolean merge_channels,
final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
......@@ -1476,6 +1476,7 @@ public class QuadCLT extends QuadCLTCPU {
threadsMax, // final int threadsMax, // maximal number of threads to launch
debugLevel); // final int globalDebugLevel);
}
ImagePlus imp_render = scene.renderFromTD (
sensor_mask, // final int sensor_mask,
merge_channels, // boolean merge_channels,
......@@ -1489,6 +1490,293 @@ public class QuadCLT extends QuadCLTCPU {
return imp_render;
}
public static ImagePlus renderGPUFromDSI(
final int sensor_mask,
final boolean merge_channels,
final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
CLTParameters clt_parameters,
double [] disparity_ref,
double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
double mb_tau, // 0.008; // time constant, sec
double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
double [][] mb_vectors, // now [2][ntiles];
double [] scene_xyz, // camera center in world coordinates. If null - no shift, no ers
double [] scene_atr, // camera orientation relative to world frame
final QuadCLT scene,
final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
final boolean toRGB,
final boolean show_nan,
String suffix,
int threadsMax,
final int debugLevel){
preRenderGPUFromDSI(
sensor_mask, // final int sensor_mask,
merge_channels, // final boolean merge_channels,
full_woi_in, // final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
clt_parameters, // CLTParameters clt_parameters,
disparity_ref, // double [] disparity_ref,
ref_pXpYD, // double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, // double [][] mb_vectors, // now [2][ntiles];
scene_xyz, // double [] scene_xyz, // camera center in world coordinates. If null - no shift, no ers
scene_atr, // double [] scene_atr, // camera orientation relative to world frame
scene, //final QuadCLT scene,
ref_scene, // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
// toRGB, // final boolean toRGB,
show_nan, // final boolean show_nan,
// suffix, // String suffix,
threadsMax, // int threadsMax,
debugLevel); // final int debugLevel)
int [] wh = (full_woi_in == null)? null: new int[]{
full_woi_in.width * GPUTileProcessor.DTT_SIZE,
full_woi_in.height * GPUTileProcessor.DTT_SIZE};
ImagePlus imp_render = scene.renderFromTD (
sensor_mask, // final int sensor_mask,
merge_channels, // boolean merge_channels,
clt_parameters, // CLTParameters clt_parameters,
clt_parameters.getColorProcParameters(scene.isAux()), //ColorProcParameters colorProcParameters,
clt_parameters.getRGBParameters(), //EyesisCorrectionParameters.RGBParameters rgbParameters,\
wh, // null, // int [] wh,
toRGB, // boolean toRGB,
false, // use_reference, // boolean use_reference
suffix); // String suffix)
return imp_render;
}
public static float [][] getTDCombo(
final int sensor_mask,
final boolean merge_channels,
final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
CLTParameters clt_parameters,
double [] disparity_ref,
double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
double mb_tau, // 0.008; // time constant, sec
double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
double [][] mb_vectors, // now [2][ntiles];
double [] scene_xyz, // camera center in world coordinates. If null - no shift, no ers
double [] scene_atr, // camera orientation relative to world frame
final QuadCLT scene,
final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
// final boolean toRGB,
final boolean show_nan,
// String suffix,
int threadsMax,
final int debugLevel){
preRenderGPUFromDSI(
sensor_mask, // final int sensor_mask,
merge_channels, // final boolean merge_channels,
full_woi_in, // final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
clt_parameters, // CLTParameters clt_parameters,
disparity_ref, // double [] disparity_ref,
ref_pXpYD, // double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
mb_tau, // double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, // double [][] mb_vectors, // now [2][ntiles];
scene_xyz, // double [] scene_xyz, // camera center in world coordinates. If null - no shift, no ers
scene_atr, // double [] scene_atr, // camera orientation relative to world frame
scene, //final QuadCLT scene,
ref_scene, // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
// toRGB, // final boolean toRGB,
show_nan, // final boolean show_nan,
// suffix, // String suffix,
threadsMax, // int threadsMax,
debugLevel); // final int debugLevel)
int [] whc = new int [3];
float [][] fclt = scene.getComboFromTD(
sensor_mask, // final int sensor_mask, // only if merge_channels
merge_channels, // final boolean merge_channels,
whc, // final int [] whc, // if int[2], will return width, height
false); // final boolean use_reference)
return fclt;
}
public static void preRenderGPUFromDSI(
final int sensor_mask,
final boolean merge_channels,
final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
CLTParameters clt_parameters,
double [] disparity_ref,
double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
// motion blur compensation
double mb_tau, // 0.008; // time constant, sec
double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
double [][] mb_vectors, // now [2][ntiles];
double [] scene_xyz, // camera center in world coordinates. If null - no shift, no ers
double [] scene_atr, // camera orientation relative to world frame
final QuadCLT scene,
final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
// final boolean toRGB,
final boolean show_nan,
// String suffix,
int threadsMax,
final int debugLevel){
double [][] pXpYD;
if (ref_pXpYD != null) { // cuas mode, ref_pXpYD defines offset reference scene
pXpYD=OpticalFlow.transformToScenePxPyD(
ref_pXpYD, // final double [][] reference_pXpYD, // invalid tiles - NaN in disparity. Should be no nulls, no NaN disparity
scene_xyz, // final double [] scene_xyz, // camera center in world (reference) coordinates
scene_atr, // final double [] scene_atr, // camera orientation relative to world (reference) frame
ref_scene, // final QuadCLT reference_QuadClt)
scene); // final QuadCLT scene_QuadClt)
scene.getErsCorrection().setupERS(); // NEW - did not help
// ref_scene.getErsCorrection().setupERS(); // just in case - setup using instance parameters - inside
} else {
if ((scene_xyz == null) || (scene_atr == null)) {
scene_xyz = new double[3];
scene_atr = new double[3];
pXpYD=OpticalFlow.transformToScenePxPyD( // now should work with offset ref_scene
full_woi_in, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
disparity_ref, // final double [] disparity_ref, // invalid tiles - NaN in disparity
scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
ref_scene, // final QuadCLT scene_QuadClt,
ref_scene, // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
threadsMax); // int threadsMax)
} else {
pXpYD=OpticalFlow.transformToScenePxPyD( // now should work with offset ref_scene
full_woi_in, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
disparity_ref, // final double [] disparity_ref, // invalid tiles - NaN in disparity
scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
scene, // final QuadCLT scene_QuadClt,
ref_scene, // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
threadsMax); // int threadsMax)
}
}
int rendered_width = scene.getErsCorrection().getSensorWH()[0];
if (full_woi_in != null) {
rendered_width = full_woi_in.width * GPUTileProcessor.DTT_SIZE;
}
boolean showPxPyD = false;
if (showPxPyD) {
int dbg_width = rendered_width/GPUTileProcessor.DTT_SIZE;
int dbg_height = pXpYD.length/dbg_width;
String [] dbg_titles = (mb_vectors!=null)?
(new String[] {"pX","pY","Disparity","mb_X","mb_Y","disparity_ref"}):
(new String[] {"pX","pY","Disparity","disparity_ref"});
double [][] dbg_img = new double [dbg_titles.length][pXpYD.length]; // 3 + ((mb_vectors!=null)? 2:0)][pXpYD.length];
for (int i = 0; i < dbg_img.length; i++) {
Arrays.fill(dbg_img[i], Double.NaN);
}
for (int nTile = 0; nTile < pXpYD.length; nTile++){
if (pXpYD[nTile] != null) {
for (int i = 0; i < pXpYD[nTile].length; i++) {
dbg_img[i][nTile] = pXpYD[nTile][i];
}
}
if (mb_vectors!=null) {
for (int i = 0; i <2; i++) {
dbg_img[3 + i][nTile] = mb_tau * mb_vectors[i][nTile];
}
}
}
dbg_img[dbg_img.length-1] = disparity_ref;
ShowDoubleFloatArrays.showArrays( // out of boundary 15
dbg_img,
dbg_width,
dbg_height,
true,
scene.getImageName()+"-pXpYD",
dbg_titles);
}
TpTask[][] tp_tasks;
if (mb_vectors!=null) {
tp_tasks = GpuQuad.setInterTasksMotionBlur( // "true" reference, with stereo actual reference will be offset
scene.getNumSensors(),
rendered_width, // should match output size, pXpYD.length
!scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
// motion blur compensation
mb_tau, // final double mb_tau, // 0.008; // time constant, sec
mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
mb_vectors, //final double [][] mb_vectors, //
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
clt_parameters.imp.disparity_corr, // 04/07/2023 //0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
threadsMax); // final int threadsMax) // maximal number of threads to launch
} else {
tp_tasks = new TpTask[1][];
tp_tasks[0] = GpuQuad.setInterTasks( // "true" reference, with stereo actual reference will be offset
scene.getNumSensors(),
rendered_width, // should match output size, pXpYD.length
!scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
clt_parameters.imp.disparity_corr, // 04/07/2023 // 0.0, // final double disparity_corr,
-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
null, // final boolean [] valid_tiles,
threadsMax); // final int threadsMax) // maximal number of threads to launch
}
scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
ImageDtt image_dtt = new ImageDtt(
scene.getNumSensors(),
clt_parameters.transform_size,
clt_parameters.img_dtt,
scene.isAux(),
scene.isMonochrome(),
scene.isLwir(),
clt_parameters.getScaleStrength(scene.isAux()),
scene.getGPU());
boolean use_reference = false;
int [] wh = (full_woi_in == null)? null: new int[]{
full_woi_in.width * GPUTileProcessor.DTT_SIZE,
full_woi_in.height * GPUTileProcessor.DTT_SIZE};
int erase_clt = show_nan ? 1:0;
// boolean test1 = true;
if (mb_vectors!=null) {// && test1) {
image_dtt.setReferenceTDMotionBlur( // change to main?
erase_clt, //final int erase_clt,
wh, // null, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
use_reference, // true, // final boolean use_reference_buffer,
tp_tasks, // final TpTask[] tp_tasks,
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
threadsMax, // final int threadsMax, // maximal number of threads to launch
debugLevel); // final int globalDebugLevel);
} else {
image_dtt.setReferenceTD( // change to main?
erase_clt, //final int erase_clt,
wh, // null, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
use_reference, // true, // final boolean use_reference_buffer,
tp_tasks[0], // final TpTask[] tp_tasks,
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
threadsMax, // final int threadsMax, // maximal number of threads to launch
debugLevel); // final int globalDebugLevel);
}
return;
}
public static double [][] getScenePxPyD(
final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
......@@ -1524,7 +1812,7 @@ public class QuadCLT extends QuadCLTCPU {
}
public static double [][] renderDoubleGPUFromDSI( // {scene}{color}{pixel}
public static double [][] renderDoubleGPUFromDSI( // TODO: update using preRenderGPUFromDSI(), similar to renderGPUFromDSI()
final int sensor_mask,
final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
CLTParameters clt_parameters,
......@@ -2325,8 +2613,215 @@ public class QuadCLT extends QuadCLTCPU {
return texture_tiles88;
}
public int getCltSize(boolean use_ref) {// per sensor, in floats
return gpuQuad.getCltSize(use_ref);
}
public int getNumSensors() { // Use QCC - this one may be null
if (gpuQuad != null) {
return gpuQuad.getNumSensors();
}
return super.getNumSensors();
}
/**
* Get individual or combined transform domain data from the GPU
* @param sensor_mask bitmask of the sensors to use
* @param merge_channels true if all sensors data will be combined in result[0], false if each
* sensor's data is output individually
* @param whc null or int[2] or int[3]. If non-null will return width, hight and number of colors
* @param use_reference set the reference buffers (false - set main buffers)
* @return two dimensional array of float: [num_cameras][data] od [1][data] if merge_channels.
*/
float [][] getComboFromTD(
final int sensor_mask, // only if merge_channels
final boolean merge_channels,
final int [] whc, // if int[2], will return width, height
final boolean use_reference){
final int [] width_height = gpuQuad.getWH(use_reference);
final int tilesX = width_height[0] / GPUTileProcessor.DTT_SIZE;
final int tilesY = width_height[1] / GPUTileProcessor.DTT_SIZE;
final int tile_size_td = 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
final int num_colors = gpuQuad.getNumColors();
final int num_tiles = tilesY*tilesX*num_colors;
if (whc != null) {
whc[0] = width_height[0];
whc[1] = width_height[1];
if (whc.length > 2) {
whc[2] = num_colors;
}
}
final float [][] fclt = gpuQuad.getCltData(
use_reference);// boolean use_ref);
if (merge_channels) {
final float [] fclt_merged = new float[ tile_size_td * num_tiles];
final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nTile = ai.getAndIncrement(); nTile < num_tiles; nTile = ai.getAndIncrement()) {
int indx = nTile * tile_size_td;
for (int i = 0; i < tile_size_td; i++) {
int num_valid = 0;
for (int ncam = 0; ncam < fclt.length; ncam++) if (((1 << ncam) & sensor_mask) != 0){
if (!Float.isNaN(fclt[ncam][indx])) {
fclt_merged[indx] += fclt[ncam][indx];
num_valid++;
}
}
if (num_valid > 0) {
fclt_merged[indx] /= num_valid;
} else {
fclt_merged[indx] = Float.NaN;
}
indx++;
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return new float[][] {fclt_merged};
}
return fclt;
}
/**
* Get individual or combined transform domain data from the GPU. Converts float to double
* @param sensor_mask bitmask of the sensors to use
* @param merge_channels true if all sensors data will be combined in result[0], false if each
* sensor's data is output individually
* @param whc null or int[2] or int[3]. If non-null will return width, hight and number of colors
* @param use_reference set the reference buffers (false - set main buffers)
* @return two dimensional array of double: [num_cameras][data] od [1][data] if merge_channels.
*/
double [][] getDoubleComboFromTD(
final int sensor_mask, // only if merge_channels
final boolean merge_channels,
final int [] whc, // if int[2], will return width, height
final boolean use_reference){
float [][] fclt= getComboFromTD(
sensor_mask, //final int sensor_mask, // only if merge_channels
merge_channels, // final boolean merge_channels,
whc, // final int [] whc, // if int[2], will return width, height
use_reference); // final boolean use_reference)
final double [][] dfclt = new double [fclt.length][fclt[0].length];
final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nChn = ai.getAndIncrement(); nChn < dfclt.length; nChn = ai.getAndIncrement()) {
for (int i = 0; i < dfclt[nChn].length; i++) {
dfclt[nChn][i] = fclt[nChn][i];
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return dfclt;
}
/**
* Set Transform Domain data from array, usually accumulated from many scenes/sensors. Can set
* individual sensor data from corresponding fclt row, or all sensors may be set from the same
* fclt[0].
* @param fclt two dimensional array of float [num_cameras][data] or [1][data]
* @param merge_channels true if all sensors will get the same data (fclt[0][])
* @param sensor_mask bitmask of the sensors to set
* @param whc null or int[2] or int[3]. If non-null will return width, hight and number of colors
* @param use_reference set the reference buffers (false - set main buffers)
*/
void setComboToTD(
final float [][] fclt,
final boolean merge_channels, // duplicate same data to all selected channels
final int sensor_mask, // only if merge_channels
final int [] whc, // if int[2], will return width, height
final boolean use_reference){
final int [] width_height = gpuQuad.getWH(use_reference);
final int num_colors = gpuQuad.getNumColors();
if (whc != null) {
whc[0] = width_height[0];
whc[1] = width_height[1];
if (whc.length > 2) {
whc[2] = num_colors;
}
}
for (int ncam = 0; ncam < fclt.length; ncam++) if (((1 << ncam) & sensor_mask) != 0){
int src_cam = merge_channels? 0: ncam;
gpuQuad.setCltData( // for testing only
ncam, // int ncam,
fclt[src_cam], // float [] fclt, //
use_reference); // boolean use_ref);
}
return;
}
/**
* Set Transform domain data from array, usually accumulated from many scenes/sensors. Can set
* individual sensor data from corresponding fclt row, or all sensors may be set from the same
* dfclt[0]. Convert from double [][] to float [][]
* @param fclt two dimensional array of double [num_cameras][data] or [1][data]
* @param sensor_mask bitmask of the sensors to set
* @param merge_channels true if all sensors will get the same data (fclt[0][])
* @param whc null or int[2] or int[3]. If non-null will return width, hight and number of colors
* @param use_reference set the reference buffers (false - set main buffers)
*/
void setComboToTD(
final double [][] dfclt,
final int sensor_mask, // only if merge_channels
final boolean merge_channels, // duplicate same data to all selected channels
final int [] whc, // if int[2], will return width, height
final boolean use_reference){
final float [][] fclt = new float [dfclt.length][dfclt[0].length];
final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nChn = ai.getAndIncrement(); nChn < fclt.length; nChn = ai.getAndIncrement()) {
for (int i = 0; i < fclt[nChn].length; i++) {
fclt[nChn][i] = (float) dfclt[nChn][i];
}
}
}
};
}
ImageDtt.startAndJoin(threads);
setComboToTD(
fclt, // final float [][] fclt,
merge_channels, // final boolean merge_channels, // duplicate same data to all selected channels
sensor_mask, // final int sensor_mask,
whc, // final int [] whc, // if int[2], will return width, height
use_reference); // final boolean use_reference)
return;
}
/*
float [][] fclt = gpuQuad.getCltData(
false);// boolean use_ref);
System.out.println("Got CLT ["+fclt.length+"]["+fclt[0].length+"]");
ShowDoubleFloatArrays.showArrays(
fclt,
gpuQuad.img_width*2,
gpuQuad.img_height*2,
true,
"fclt_raw"); // , dbg_titles);
float [][] pfclt = gpuQuad.presentCltData(false);
ShowDoubleFloatArrays.showArrays(
pfclt,
gpuQuad.img_width*2,
gpuQuad.img_height*2,
true,
"fclt"); // , dbg_titles);
*/
// add similar for textures
public ImagePlus renderFromTD (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment