Skip to content

I
imagej-elphel
Commit c899aa2b authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

CLAUDE: Wire DNN front-end into CuasDetectRT (-DNN field, all scenes) 

When curt_dnn_model is set, run the ONNX DNN per-pixel inference over
curt_save_select across all available scenes (not just the first few),
saving -DNN-RECT / -DNN-HYPER-RECT. Slice labels carry timestamp + absolute
frame index for matching to real targets. New curt_dnn_thresh (6 sites in
IntersceneMatchParameters) zeroes the (Vx,Vy,s) field below confidence s -
display/FP-suppression only. inferROI tiles the ROI batch (CHUNK) to bound
ONNX Runtime activation memory.
Co-authored-by: 's avatarClaude Opus 4.8 (1M context) <noreply@anthropic.com>
parent f19ac162
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 0 deletions
src/main/java/com/elphel/imagej/cuas/rt/CuasDetectRT.java
View file @ c899aa2b
......@@ -1030,6 +1030,58 @@ public class CuasDetectRT {
} // By Claude on 06/11/2026
} // By Claude on 06/11/2026
} // By Claude on 06/11/2026
// DNN front-end (-DNN): if curt_dnn_model is set, run the trained FCN per-pixel // By Claude on 06/13/2026
// (stride-1, matching training) over the ROI -> (Vx,Vy,s) field = softmax(vel)*s,
// same shape as the C5P output, saved as -DNN-RECT / -DNN-HYPER-RECT. N=8 frames
// (training). Reuses curt_c5_levels for level gating. Recurrent feed is the next step
// (the field is [0,1]-scaled, unlike the C5P matched-filter response - needs rescale).
if (!clt_parameters.imp.curt_dnn_model.isEmpty() && (curt_save_select != null)) { // By Claude on 06/13/2026
final int N_dnn = 8; // By Claude on 06/13/2026
final int vr_dnn = clt_parameters.imp.curt_vel_radius; // By Claude on 06/13/2026
int [] c5_levels_dnn = clt_parameters.imp.curt_c5_levels; // By Claude on 06/13/2026
try { // By Claude on 06/13/2026
CuasDnnInfer dnn = new CuasDnnInfer(clt_parameters.imp.curt_dnn_model, N_dnn); // By Claude on 06/13/2026
for (int nlev = 0; nlev < pyramid_levels; nlev++) { // By Claude on 06/13/2026
boolean lev_sel = (c5_levels_dnn == null) || (c5_levels_dnn.length == 0); // By Claude on 06/13/2026
if (!lev_sel) for (int l : c5_levels_dnn) if (l == nlev) { lev_sel = true; break; } // By Claude on 06/13/2026
if (!lev_sel) continue; // By Claude on 06/13/2026
double [][] framesD = dpixels_pyramid[nlev]; // By Claude on 06/13/2026
if (framesD == null) continue; // By Claude on 06/13/2026
int num = Math.max(0, framesD.length - N_dnn + 1); // all available scenes // By Claude on 06/13/2026
if (num == 0) continue; // By Claude on 06/13/2026
double dnn_thresh = clt_parameters.imp.curt_dnn_thresh; // By Claude on 06/13/2026
double [][][][] dnn_roi = new double [num][][][]; // By Claude on 06/13/2026
String [] ts_dnn = new String [num]; // By Claude on 06/13/2026
System.out.println(now()+" detectTargets(): DNN inferROI LEV"+nlev+" ("+num+" scenes, "+ // By Claude on 06/13/2026
(curt_save_select.width*curt_save_select.height)+" px/scene, thresh="+dnn_thresh+")"); // By Claude on 06/13/2026
for (int n5d = 0; n5d < num; n5d++) { // By Claude on 06/13/2026
int newest = n5d + N_dnn - 1; // absolute newest-frame index in framesD // By Claude on 06/13/2026
float [][] window = new float [N_dnn][]; // By Claude on 06/13/2026
for (int h = 0; h < N_dnn; h++) { // By Claude on 06/13/2026
double [] src = framesD[newest - h]; // h=0 newest // By Claude on 06/13/2026
float [] fw = new float [src.length]; // By Claude on 06/13/2026
for (int k = 0; k < src.length; k++) fw[k] = (float) src[k]; // By Claude on 06/13/2026
window[h] = fw; // By Claude on 06/13/2026
} // By Claude on 06/13/2026
dnn_roi[n5d] = dnn.inferROI(window, width, height, curt_save_select, vr_dnn, dnn_thresh); // By Claude on 06/13/2026
ts_dnn[n5d] = ts_pyramid[nlev][newest] + " f"+newest; // timestamp + abs frame index for matching // By Claude on 06/13/2026
System.out.println(now()+" DNN scene "+(n5d+1)+"/"+num+" (frame "+newest+", "+ts_pyramid[nlev][newest]+") done"); // By Claude on 06/13/2026
} // By Claude on 06/13/2026
String title_dnn = title_conv5d+"-DNN"+((nlev > 0)?("-LEV"+nlev):""); // By Claude on 06/13/2026
if (curt_save_c5rect) { // By Claude on 06/13/2026
QuadCLTCPU.saveImagePlusInDirectory(cuasRTUtils.showConvKernel5d( // By Claude on 06/13/2026
dnn_roi, curt_save_select, ts_dnn, title_dnn+"-RECT"), getModelDirectory()); // By Claude on 06/13/2026
} // By Claude on 06/13/2026
if (clt_parameters.imp.curt_save_c5hyper) { // By Claude on 06/13/2026
QuadCLTCPU.saveImagePlusInDirectory(cuasRTUtils.showConvKernel5dHyperRect( // By Claude on 06/13/2026
dnn_roi, curt_save_select, ts_dnn, title_dnn+"-HYPER-RECT"), getModelDirectory()); // By Claude on 06/13/2026
} // By Claude on 06/13/2026
} // By Claude on 06/13/2026
dnn.close(); // By Claude on 06/13/2026
} catch (Exception e) { // By Claude on 06/13/2026
System.out.println("DNN inference failed: "+e); e.printStackTrace(); // By Claude on 06/13/2026
} // By Claude on 06/13/2026
} // By Claude on 06/13/2026
System.out.println(now()+" detectTargets(): done"); // By Claude on 06/11/2026
return null;
/*
......
src/main/java/com/elphel/imagej/cuas/rt/CuasDnnInfer.java
View file @ c899aa2b
......@@ -9,6 +9,7 @@ package com.elphel.imagej.cuas.rt;
// URL; remote specs are fetched to ~/.cache/c5p_dnn/ together with the external-data sibling
// (model.onnx + model.onnx.data, which torch 2.9's exporter splits).
import java.awt.Rectangle;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
......@@ -62,6 +63,58 @@ public class CuasDnnInfer implements AutoCloseable {
}
}
/** Run the FCN per-pixel (stride-1, matching the ±0.5px-center training) over an ROI of the # By Claude on 06/13/2026
* conditioned N-frame stack. frames=[N][width*height], newest first. Extracts a 24x24 patch
* centered on each ROI pixel (zero-padded at image edges), batches them, runs ORT once, and
* returns the C5P-shaped field [roi_npix][1][vdim*vdim] = softmax(vel)*s (confidence-gated
* velocity posterior per pixel) - plugs straight into showConvKernel5d / the recurrent layer. */
public double[][][] inferROI(float[][] frames, int width, int height, Rectangle roi, int velRadius, double sThresh) throws Exception {
final int N = frames.length;
final int P = 24, half = P / 2; // receptive field == training patch
final int vdim = 2 * velRadius + 1, nvel = vdim * vdim;
final int rw = roi.width, rh = roi.height, npix = rw * rh;
final int CHUNK = 2048; // tile the batch so ORT activations stay small (B*32*22*22)
double[][][] field = new double[npix][1][nvel];
for (int base = 0; base < npix; base += CHUNK) {
int bs = Math.min(CHUNK, npix - base);
FloatBuffer fb = FloatBuffer.allocate(bs * N * P * P);
for (int q = 0; q < bs; q++) {
int p = base + q;
int cy = roi.y + (p / rw), cx = roi.x + (p % rw);
for (int n = 0; n < N; n++) {
float[] fr = frames[n];
for (int j = 0; j < P; j++) {
int iy = cy + j - half;
boolean yok = (iy >= 0 && iy < height);
for (int i = 0; i < P; i++) {
int ix = cx + i - half;
fb.put((yok && ix >= 0 && ix < width) ? fr[iy * width + ix] : 0f);
}
}
}
}
fb.rewind();
try (OnnxTensor in = OnnxTensor.createTensor(env, fb, new long[] {bs, N, P, P});
OrtSession.Result res = session.run(Collections.singletonMap(inputName, in))) {
float[][][][] out = (float[][][][]) res.get(0).getValue(); // [bs, outCh, 1, 1]
for (int q = 0; q < bs; q++) {
float[][][] op = out[q];
float s = sigmoid(op[0][0][0]);
if (s < sThresh) continue; // below confidence -> leave field 0 // By Claude on 06/13/2026
double mx = Double.NEGATIVE_INFINITY;
for (int v = 0; v < nvel; v++) mx = Math.max(mx, op[1 + v][0][0]);
double sum = 0; double[] e = new double[nvel];
for (int v = 0; v < nvel; v++) { e[v] = Math.exp(op[1 + v][0][0] - mx); sum += e[v]; }
double[] fv = field[base + q][0];
for (int v = 0; v < nvel; v++) fv[v] = (e[v] / sum) * s;
}
}
if ((((base / CHUNK) & 7) == 7) || (base + CHUNK >= npix)) // periodic progress // By Claude on 06/13/2026
System.out.println(" DNN inferROI " + Math.min(base + CHUNK, npix) + "/" + npix + " px");
}
return field;
}
public static float sigmoid(float x) { return (float) (1.0 / (1.0 + Math.exp(-x))); }
@Override public void close() throws Exception { session.close(); }
......@@ -113,6 +166,24 @@ public class CuasDnnInfer implements AutoCloseable {
}
public static void main(String[] args) throws Exception {
if (args[0].equals("roitest")) { // roitest model testvec_in.bin testvec_out.bin
int N = 8, P = 24, vr = 5, nvel = (2 * vr + 1) * (2 * vr + 1);
float[] flat = readFloatsLE(args[2]);
float[][] frames = new float[N][P * P]; // one P*P "image" per frame
for (int n = 0; n < N; n++) System.arraycopy(flat, n * P * P, frames[n], 0, P * P);
try (CuasDnnInfer infer = new CuasDnnInfer(args[1], N)) {
// 1x1 ROI at the image center -> extracted patch == the full P*P frame
double[][][] field = infer.inferROI(frames, P, P, new Rectangle(P / 2, P / 2, 1, 1), vr, 0.0);
float[] exp = readFloatsLE(args[3]); // raw [124]; build softmax(vel)*s
float s = sigmoid(exp[0]); double mx = -1e30, sum = 0; double[] e = new double[nvel];
for (int v = 0; v < nvel; v++) mx = Math.max(mx, exp[1 + v]);
for (int v = 0; v < nvel; v++) { e[v] = Math.exp(exp[1 + v] - mx); sum += e[v]; }
double maxd = 0; for (int v = 0; v < nvel; v++) maxd = Math.max(maxd, Math.abs(field[0][0][v] - e[v] / sum * s));
System.out.printf("inferROI roitest: max abs diff vs single-patch softmax*s = %.6g %s%n",
maxd, maxd < 1e-4 ? "PASS" : "MISMATCH");
}
return;
}
// args: model(path/spec) testvec_in.bin [testvec_out.bin] [N H W]
String model = args[0], invec = args[1];
String outvec = args.length > 2 ? args[2] : null;
......
src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java
View file @ c899aa2b
......@@ -1178,6 +1178,7 @@ min_str_neib_fpn 0.35
public double curt_c5_white_sp = 0.5; // whitening spatial strength a (3-tap unsharp (1+2a,-a,-a) in x and y); 0 - no spatial sharpening // By Claude on 06/13/2026
public double curt_c5_white_vel = 0.0; // whitening velocity strength a (3-tap unsharp in vx and vy); 0 - no velocity sharpening // By Claude on 06/13/2026
public String curt_dnn_model = ""; // C5P DNN front-end model (ONNX): empty = disabled; local path, scp user@host:path, or http(s) URL fetched to cache; overrides bundled resource (mirrors tile_processor_gpu) // By Claude on 06/13/2026
public double curt_dnn_thresh = 0.0; // DNN display threshold: zero the (Vx,Vy,s) field where detection s < this (0 - show all); cosmetic FP-suppression - real-clutter FPs need a retrain with real-bg negatives, not a threshold // By Claude on 06/13/2026
public boolean curt_synth_src = true; // default set for the synthetic B-measurement experiment (set false for real-data runs); reads *-CUAS-SYNTHETIC-CUAS.tiff, output titles get -SYNTH // By Claude on 06/12/2026
public double curt_synth_scale = 5.0; // synthetic target peak, counts (synthetic file is peak-1 normalized; scaled at load) // By Claude on 06/12/2026
public boolean curt_synth_bg = true; // add the real *-CUAS-MERGED-CUAS.tiff scene under the synthetic targets (label-matched frames); false - clean targets only // By Claude on 06/12/2026
......@@ -3519,6 +3520,8 @@ min_str_neib_fpn 0.35
"Velocity unsharp strength a: same 3-tap in vx and vy within each pixel's velocity grid. Sharpens the velocity-cell spread. 0 - off (start spatial-only)."); // By Claude on 06/13/2026
gd.addStringField ("C5P DNN model (ONNX)", this.curt_dnn_model, 60, // By Claude on 06/13/2026
"Trained DNN front-end model location. Empty = disabled (use matched-filter/posterior path). Local path, scp user@host:/path (fetched to cache), or http(s) URL. Overrides any bundled resource - same default-vs-override scheme as the GPU kernel sources."); // By Claude on 06/13/2026
gd.addNumericField("DNN confidence threshold", this.curt_dnn_thresh, 6,8,"", // By Claude on 06/13/2026
"Zero the DNN (Vx,Vy,s) field where detection confidence s < this (0 = show all). Display/FP-suppression only - real-clutter false positives need a retrain with real-background negatives, not a threshold."); // By Claude on 06/13/2026
gd.addCheckbox ("Use synthetic input", this.curt_synth_src, // By Claude on 06/11/2026
"Read *-CUAS-SYNTHETIC-CUAS.tiff (generated test targets) instead of *-CUAS-MERGED-CUAS.tiff from the same model directory; all output titles get a -SYNTH mark."); // By Claude on 06/11/2026
gd.addNumericField("Synthetic target peak", this.curt_synth_scale, 6,8,"counts", // By Claude on 06/12/2026
......@@ -5064,6 +5067,7 @@ min_str_neib_fpn 0.35
this.curt_c5_white_sp = gd.getNextNumber(); // By Claude on 06/13/2026
this.curt_c5_white_vel = gd.getNextNumber(); // By Claude on 06/13/2026
this.curt_dnn_model = gd.getNextString().trim(); // By Claude on 06/13/2026
this.curt_dnn_thresh = gd.getNextNumber(); // By Claude on 06/13/2026
this.curt_synth_src = gd.getNextBoolean(); // By Claude on 06/11/2026
this.curt_synth_scale = gd.getNextNumber(); // By Claude on 06/12/2026
this.curt_synth_bg = gd.getNextBoolean(); // By Claude on 06/12/2026
......@@ -6429,6 +6433,7 @@ min_str_neib_fpn 0.35
properties.setProperty(prefix+"curt_c5_white_sp", this.curt_c5_white_sp+""); // double // By Claude on 06/13/2026
properties.setProperty(prefix+"curt_c5_white_vel", this.curt_c5_white_vel+""); // double // By Claude on 06/13/2026
properties.setProperty(prefix+"curt_dnn_model", this.curt_dnn_model); // String // By Claude on 06/13/2026
properties.setProperty(prefix+"curt_dnn_thresh", this.curt_dnn_thresh+""); // double // By Claude on 06/13/2026
properties.setProperty(prefix+"curt_synth_src", this.curt_synth_src+""); // boolean // By Claude on 06/11/2026
properties.setProperty(prefix+"curt_synth_scale", this.curt_synth_scale+""); // double // By Claude on 06/12/2026
properties.setProperty(prefix+"curt_synth_bg", this.curt_synth_bg+""); // boolean // By Claude on 06/12/2026
......@@ -6829,6 +6834,7 @@ min_str_neib_fpn 0.35
if (properties.getProperty(prefix+"curt_c5_white_sp")!=null) this.curt_c5_white_sp=Double.parseDouble(properties.getProperty(prefix+"curt_c5_white_sp")); // By Claude on 06/13/2026
if (properties.getProperty(prefix+"curt_c5_white_vel")!=null) this.curt_c5_white_vel=Double.parseDouble(properties.getProperty(prefix+"curt_c5_white_vel")); // By Claude on 06/13/2026
if (properties.getProperty(prefix+"curt_dnn_model")!=null) this.curt_dnn_model=(String) properties.getProperty(prefix+"curt_dnn_model"); // By Claude on 06/13/2026
if (properties.getProperty(prefix+"curt_dnn_thresh")!=null) this.curt_dnn_thresh=Double.parseDouble(properties.getProperty(prefix+"curt_dnn_thresh")); // By Claude on 06/13/2026
if (properties.getProperty(prefix+"curt_synth_src")!=null) this.curt_synth_src=Boolean.parseBoolean(properties.getProperty(prefix+"curt_synth_src")); // By Claude on 06/11/2026
if (properties.getProperty(prefix+"curt_synth_scale")!=null) this.curt_synth_scale=Double.parseDouble(properties.getProperty(prefix+"curt_synth_scale")); // By Claude on 06/12/2026
......@@ -9111,6 +9117,7 @@ min_str_neib_fpn 0.35
imp.curt_c5_white_sp = this.curt_c5_white_sp; // By Claude on 06/13/2026
imp.curt_c5_white_vel = this.curt_c5_white_vel; // By Claude on 06/13/2026
imp.curt_dnn_model = this.curt_dnn_model; // By Claude on 06/13/2026
imp.curt_dnn_thresh = this.curt_dnn_thresh; // By Claude on 06/13/2026
imp.curt_synth_src = this.curt_synth_src; // By Claude on 06/11/2026
imp.curt_synth_scale = this.curt_synth_scale; // By Claude on 06/12/2026
imp.curt_synth_bg = this.curt_synth_bg; // By Claude on 06/12/2026
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment