Skip to content

I
imagej-elphel
Commit 22a12ccf authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

Combining moving targets with running average

parent 6dd7be11
Hide whitespace changes
Inline Side-by-side
Showing with 344 additions and 22 deletions
src/main/java/com/elphel/imagej/cuas/CuasMotion.java
View file @ 22a12ccf
package com.elphel.imagej.cuas;
import java.awt.Rectangle;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -26,9 +27,14 @@ public class CuasMotion {
final static private int INDX_FRAC = 3;
final static private int INDX_SPEED = 4; // calculated separately
final static private int INDX_CONFIDENCE = 5; // calculated separately
final static String [] VF_TOP_TITLES = {"vX","vY","strength","fraction","speed", "confidence"};
final static public int TARGET_X = 0;
final static public int TARGET_Y = 1;
final static public int TARGET_VX = 2;
final static public int TARGET_VY = 3;
final static public int TARGET_STRENGTH = 4;
final static public int TARGET_LENGTH = TARGET_STRENGTH+1;
private final GPUTileProcessor gpuTileProcessor;
private CLTParameters clt_parameters=null;
......@@ -697,21 +703,45 @@ public class CuasMotion {
cuasMotion.gpu_max_height,
true,
title_accumulated+"-TARGETS5x5", // "-corr2d"+"-"+frame0+"-"+frame1+"-"+corr_pairs,
scene_titles); // titles_accum);
slice_titles); // titles_accum);
/*
float [][] fpixels_accumulated_filtered = getTargetImages(
targets_lma_combo, // final double [][][] vector_fields, // centers , just null/not null
accum, // final float [][] accum_data, // should be around 0, no low-freq
cuasMotion.tilesX); // final int tilesX)
double mask_width = 9;
double mask_blur = 3;
boolean round = false;
double velocity_scale = 1.0/corr_offset;
double [][][] targets60hz = new double [fpixels.length][][];
boolean ra_background = true;
float [][] background = fpixels;
if (ra_background) {
background = runningAverage(
fpixels, // final float [][] fpixels,
corr_pairs, // final int ra_length,
cuasMotion.gpu_max_width); // final int width)
}
float [][] replaced_targets = cuasMotion.shiftAndRenderTargets(
clt_parameters, // CLTParameters clt_parameters,
mask_width, // final double mask_width,
mask_blur, // final double mask_blur,
round, // final boolean round,
fpixels_accumulated5x5, // final float [][] target_keyframes,
extended_vf_sequence, // final double [][][] vector_field,
targets_lma_combo, // final double [][][] target_positions,
background, // final float [][] background, // background image
frame0, // final int frame0,
corr_step, // final int frame_step,
velocity_scale, // final double velocity_scale, // 1.0/(disparity in frames)
targets60hz, // final double [][][] targets60hz,
false, // final boolean batch_mode,
debugLevel); // final int debugLevel)
ShowDoubleFloatArrays.showArrays(
fpixels_accumulated_filtered,
replaced_targets,
cuasMotion.gpu_max_width,
cuasMotion.gpu_max_height,
true,
title_acc_targets+"-LMA-TEST-COMBO", // "-corr2d"+"-"+frame0+"-"+frame1+"-"+corr_pairs,
title_accumulated+"-REPLACED-TARGETS", // "-corr2d"+"-"+frame0+"-"+frame1+"-"+corr_pairs,
scene_titles); // titles_accum);
*/
continue;
}
......@@ -920,7 +950,7 @@ public class CuasMotion {
cuasMotion.gpu_max_height,
true,
title_accumulated+"-n"+niter, // "-corr2d"+"-"+frame0+"-"+frame1+"-"+corr_pairs,
scene_titles); // titles_accum);
slice_titles); // titles_accum);
}
// replace center frames with the accumulated ones
for (int nseq = 0; nseq < fpixels_accumulated.length; nseq++){
......@@ -1063,7 +1093,7 @@ public class CuasMotion {
cuasMotion.gpu_max_height,
true,
title_acc_targets+"-n"+niter, // "-corr2d"+"-"+frame0+"-"+frame1+"-"+corr_pairs,
scene_titles); // titles_accum);
slice_titles); // titles_accum);
}
if (debugLevel > -4) {
System.out.println("Iteration "+niter+" DONE.");
......@@ -1150,7 +1180,7 @@ public class CuasMotion {
cuasMotion.gpu_max_height,
true,
title_accumulated+"-n"+niter, // "-corr2d"+"-"+frame0+"-"+frame1+"-"+corr_pairs,
scene_titles); // titles_accum);
slice_titles); // titles_accum);
}
// replace center frames with the accumulated ones
for (int nseq = 0; nseq < fpixels_accumulated.length; nseq++){
......@@ -2287,6 +2317,56 @@ public class CuasMotion {
}
return;
}
/**
* Shift keyframe according to the vector_field, render and return result image. Does not need to reload keyframe image
* if it is already loaded in the GPU
* @param clt_parameters all the parameters
* @param fkeyframe keyframe data or null (if keyframe did not change)
* @param vector_field [scene number][tile number] {vx, vy, ...}. nulls for empty tiles
* @param erase -1 - no erase, 0 - to 0, 1 - to NaN
* @param batch_mode disables graphics debug in batch mode
* @param offset_scale frame offset to multiply vX, vY when calculating total tile offsets
* @return
*/
public float [] renderMovingKeyframe(
CLTParameters clt_parameters,
float [] fkeyframe, // if null, will assume it is already in the GPU
final double [][] vector_field,
final int erase, // -1 - no, 0 - to 0, 1 - to NaN
final boolean batch_mode,
final double offset_scale) {
int [] wh = {gpu_max_width, gpu_max_height};
TpTask [] tp_tasks = GpuQuad.setRectilinearMovingTasks(
vector_field, // final double [][] vector_field,
offset_scale, // final double offset_scale,
0.0, // final double magnitude_scale,
tilesX); // final int tilesX)
image_dtt.setRectilinearReferenceTD(
erase, // final int erase_clt,
fkeyframe, // final float [] fpixels_ref,
wh, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
false, // final boolean use_reference_buffer,
tp_tasks, // final TpTask[] tp_tasks,
clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
batch_mode? -3: debugLevel); // final int globalDebugLevel)
boolean frame_debug = false;
if (frame_debug) {
renderFromTD (
false, // boolean use_reference,
"render-from-TD"); // String suffix)
}
float [] rendered_frame = floatFromTD(false);
return rendered_frame;
}
public TDCorrTile [] correlatePair(
CLTParameters clt_parameters,
......@@ -2512,6 +2592,244 @@ public class CuasMotion {
return frames_accum;
}
/**
* Generate target images
* @param clt_parameters
* @param mask_width mask width around the target
* @param mask_blur mask transition width
* @param round false - square (no blur), true - round
* @param target_keyframes accumulated target images
* @param vector_field [scene][tile][] first two elements - targets vX, vY in pixels per frame. Empty are nulls, same velocities for 5x5
* @param target_positions [scene][tile][] target positions of the targets relative to the tile center
* @param background static scenes to be overlaid by the targets
* @param frame0 frame number of the background corresponding to the first keyframe
* @param frame_step numer of frames between key frames
* @param velocity_scale velocity scale to apply to vX, vY. It is equal to 1.0/corr_ofset, where corr_offset is disparity in frames
* used for vector field generation
* @param targets60hz target data for each 60hz scene
* @param batch_mode batch mode - disable graphic debugging
* @param debugLevel debug level
* @return float [][] rendered
*/
public float [][] shiftAndRenderTargets(
CLTParameters clt_parameters,
final double mask_width,
final double mask_blur,
final boolean round,
final float [][] target_keyframes,
final double [][][] vector_field,
final double [][][] target_positions,
final float [][] background, // background image
final int frame0,
final int frame_step,
final double velocity_scale, // 1.0/(disparity in frames)
final double [][][] targets60hz,
final boolean batch_mode,
final int debugLevel) {
final float [][] fpix_out = new float [background.length][];
for (int i = 0; i < fpix_out.length; i++) {
fpix_out[i]= background[i].clone();
}
final int tileSize = GPUTileProcessor.DTT_SIZE;
final int tileSize2 = 2 * tileSize;
final int half_step0 = -(frame_step+1)/2;
final int half_step1 = frame_step + half_step0;
final int num_scenes = background.length;
final int erase = 0; // probably not needed
int [] poffs = new int[1];
final double [] mask = createTargetMask (
mask_width, // final double mask_width,
mask_blur, // final double mask_blur,
round, // final boolean round,
poffs); // final int [] offs)
final int offs = poffs[0];
final int isize = 2* offs +1;
final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0);
for (int nseq = 0; nseq < vector_field.length; nseq++) {
int fnseq = nseq;
int frame_center = frame0 + nseq * frame_step;
float [] keyframe = target_keyframes[nseq];
ArrayList<Integer> target_list = new ArrayList<Integer>();
for (int ntile = 0; ntile < target_positions[nseq].length; ntile++) if (target_positions[nseq][ntile] !=null){
target_list.add(ntile);
}
final int[] targets = target_list.stream().mapToInt(Integer::intValue).toArray();
for (int dscene = half_step0; dscene < half_step1; dscene ++) {
final int fdscene = dscene;
int nscene = frame_center + dscene;
if ((nscene >=0) && (nscene < num_scenes)) {
double [][] targets_data = new double[targets.length][TARGET_LENGTH];
float [] frame = renderMovingKeyframe(
clt_parameters, // CLTParameters clt_parameters,
keyframe, // float [] fkeyframe, // if null, will assume it is already in the GPU
vector_field[nseq], // final double [][] vector_field,
erase, //final int erase, // -1 - no, 0 - to 0, 1 - to NaN
batch_mode, // final boolean batch_mode,
-dscene*velocity_scale); // final double offset_scale);
ai.set(0);
// previous renderMovingKeyframe() uses GPU, can not use threads
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nTarget = ai.getAndIncrement(); nTarget < targets.length; nTarget = ai.getAndIncrement()) {
int ntile = targets[nTarget];
double [] vvector = vector_field[fnseq][ntile];
double [] target_pos = target_positions[fnseq][ntile];
int tileX = ntile % tilesX;
int tileY = ntile / tilesX;
double xc = tileSize * tileX + tileSize/2;
double yc = tileSize * tileY + tileSize/2;
double xtk = xc + target_pos[CuasMotionLMA.RSLT_X];
double ytk = yc + target_pos[CuasMotionLMA.RSLT_Y];
double dx = vvector[INDX_VX] * fdscene * velocity_scale;
double dy = vvector[INDX_VY] * fdscene * velocity_scale;
targets_data[nTarget][TARGET_X] = xtk + dx;
targets_data[nTarget][TARGET_Y] = ytk + dy;
targets_data[nTarget][TARGET_VX] = vvector[INDX_VX];
targets_data[nTarget][TARGET_VY] = vvector[INDX_VY];
targets_data[nTarget][TARGET_STRENGTH] = target_pos[CuasMotionLMA.RSLT_A];
int pxl = (int) Math.round(targets_data[nTarget][TARGET_X]) - offs;
int pyt = (int) Math.round(targets_data[nTarget][TARGET_Y]) - offs;
for (int ym = 0; ym < isize; ym++) {
int py = pyt + ym;
if ((py >= 0) && (py < gpu_max_height)) {
for (int xm = 0; xm < isize; xm++) {
int px = pxl + xm;
if ((px >= 0) && (px < gpu_max_width)) {
int pix = py * gpu_max_width + px;
int pix_mask = ym * isize + xm;
if ((mask[pix_mask] > 0) && !Double.isNaN(frame[pix])) {
if (Float.isNaN(fpix_out[nscene][pix])) {
fpix_out[nscene][pix] = frame[pix];
} else {
double a = mask[pix_mask];
fpix_out[nscene][pix] = (float) ((1 - a) * fpix_out[nscene][pix] + a * frame[pix]);
}
}
}
}
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
if (targets60hz != null) {
targets60hz[nscene] = targets_data;
}
keyframe = null; // to prevent additional transfers to the GPU
}
}
}
return fpix_out;
}
public static float [][] runningAverage(
final float [][] fpixels,
final int ra_length,
final int width){
final int num_scenes = fpixels.length;
final int num_pixels = fpixels[0].length;
final int height = num_pixels/width;
final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0);
final float [][] out_pix = new float [num_scenes][num_pixels];
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
for (int nSeq = ai.getAndIncrement(); nSeq < num_scenes; nSeq = ai.getAndIncrement()) {
Arrays.fill(out_pix[nSeq],Float.NaN);
}
}
};
}
ImageDtt.startAndJoin(threads);
ai.set(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
float [] line_ra = new float[width];
for (int nLine = ai.getAndIncrement(); nLine < height; nLine = ai.getAndIncrement()) {
Arrays.fill(line_ra, 0.0f);
int pix0 = nLine* width;
int head = 0;
int tail = 0;
for (; (head < num_scenes) || (tail < num_scenes);) {
if (head < num_scenes) {
float [] fpixels_head = fpixels[head];
int pix = pix0;
for (int x = 0; x < width; x++) {
line_ra[x] += fpixels_head[pix++];
}
head++;
}
if ((tail < (head - ra_length)) || (tail >= (num_scenes - ra_length))) {
float [] fpixels_tail = fpixels[tail];
int pix = pix0;
for (int x = 0; x < width; x++) {
line_ra[x] -= fpixels_tail[pix++];
}
tail++;
}
int avg_len = head - tail;
int avg_pos = (head + tail) / 2;
if ((avg_pos < num_scenes) && (((avg_len + ra_length) % 2 == 0 ) || (head == 1))) { // update output array
int pix = pix0;
for (int x = 0; x < width; x++) {
out_pix[avg_pos][pix++] = line_ra[x]/avg_len;
}
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return out_pix;
}
private static double [] createTargetMask (
final double mask_width,
final double mask_blur,
final boolean round,
final int [] offs) {
int iwidth = 2*(((int) Math.ceil(mask_width))/2) + 1; // minimal odd
double [] mask = new double [iwidth*iwidth];
double r1 = mask_width/2;
double r0 = r1 - mask_blur;
double r02 = r0*r0;
double r12 = r1*r1;
int c = iwidth/2;
if (offs != null) {
offs[0] = c;
}
if (!round) {
Arrays.fill(mask, 1.0);
} else {
for (int i = 0; i < mask.length; i++) {
double x = (i % iwidth) - c;
double y = (i / iwidth) - c;
double r2 = x*x+y*y;
if (r2 < r02) {
mask[i] = 1.0;
} else if (r2 >= r12) {
mask[i] = 0.0;
} else if (mask_blur > 0){
double r = Math.sqrt(r2);
double a = Math.PI * (r - r0) / mask_blur;
mask[i] = 0.5 * (1.0 + Math.cos(a));
}
}
}
return mask;
}
public void renderFromTD (
......@@ -2570,4 +2888,7 @@ public class CuasMotion {
}
src/main/java/com/elphel/imagej/gpu/GpuQuad.java
View file @ 22a12ccf
......@@ -4725,8 +4725,8 @@ public class GpuQuad{ // quad camera description
* works with the negative scales, so the result will be a negative image in the TD.
* @param vector_field sparse array of motion vectors (may be longer, only Vx and Vy used). Null elements
* are allowed, they will be skipped, resultin in null TpTask elements.
* @param offset_scale multiply all vectors by this value when calculatingpixel offsets
* @param magnitude_scale Scale data for accumulation (here positive, will be negated
* @param offset_scale multiply all vectors by this value when calculating pixel offsets
* @param magnitude_scale Scale data for accumulation (here positive, will be negated). If 0 - will use scale=1.0 (no accumulation)
* @param image_width image width in tiles (80 for 640-wide images).
* @return condensed array of TpTask
*/
......@@ -4735,7 +4735,7 @@ public class GpuQuad{ // quad camera description
final double offset_scale,
final double magnitude_scale,
final int tilesX) {
final float fmagnitude_scale = (float) -magnitude_scale;
final float fmagnitude_scale = (magnitude_scale==0)? 1.0f : ((float) -magnitude_scale);
final int tiles = vector_field.length;
final TpTask [] tp_tasks_full = new TpTask [tiles];
final Thread[] threads = ImageDtt.newThreadArray();
......
src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java
View file @ 22a12ccf
......@@ -1495,7 +1495,7 @@ public class ImageDtt extends ImageDttCPU {
public void setRectilinearReferenceTD(
final int erase_clt,
final float [] fpixels_ref,
final float [] fpixels_ref, // if null, assumes GPU memory is already loaded
final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
final boolean use_reference_buffer,
......@@ -1516,14 +1516,15 @@ public class ImageDtt extends ImageDttCPU {
lpf_rgb,
globalDebugLevel > 2);
// gpuQuad.printConstMem("lpf_data", true);
gpuQuad.setTasks( // copy tp_tasks to the GPU memory
tp_tasks, // TpTask [] tile_tasks,
false, // use_aux); // boolean use_aux)
imgdtt_params.gpu_verify); // boolean verify
gpuQuad.setBayerImage(
fpixels_ref, // float [] bayer_image,
0); // int ncam)
if (fpixels_ref != null) {
gpuQuad.setBayerImage(
fpixels_ref, // float [] bayer_image,
0); // int ncam)
}
// allocate before execConvertDirect, so it will not be modified
boolean allocated = gpuQuad.reAllocateClt(
wh, // int [] wh,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment