Skip to content

I
imagej-elphel
Commit a53575f5 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

docs: Add U of U FOPEN DNN integration plan to project roadmap

parent 69bd7ae6
Hide whitespace changes
Inline Side-by-side
Showing with 96 additions and 25 deletions
docs/project-details.md
View file @ a53575f5
...@@ -562,6 +562,7 @@ This section captures the latest validated state before pausing Global LMA work ...@@ -562,6 +562,7 @@ This section captures the latest validated state before pausing Global LMA work
- **Adaptive Integration Windows:** Implement a competitive selection mechanism (e.g., 1x vs 4x integration) where the pipeline automatically selects the motion vector that yields the highest Contrast-to-Noise Ratio (CNR), rather than relying on hardcoded thresholds. - **Adaptive Integration Windows:** Implement a competitive selection mechanism (e.g., 1x vs 4x integration) where the pipeline automatically selects the motion vector that yields the highest Contrast-to-Noise Ratio (CNR), rather than relying on hardcoded thresholds.
- **Fat Zero Auto-Scaling:** Automatically scale `cuas_rng_fz` (fat zero) down proportionally to the length of the temporal integration (longer integrations lower the stochastic noise floor, allowing closer-to-pure phase correlation). - **Fat Zero Auto-Scaling:** Automatically scale `cuas_rng_fz` (fat zero) down proportionally to the length of the temporal integration (longer integrations lower the stochastic noise floor, allowing closer-to-pure phase correlation).
- **Acceleration Compensation:** Refine the "Virtual Moving Camera" model to handle non-linear motion (like U-turns) that currently "squash" correlation peaks during long integrations. - **Acceleration Compensation:** Refine the "Virtual Moving Camera" model to handle non-linear motion (like U-turns) that currently "squash" correlation peaks during long integrations.
- **Hybrid Classical/DNN Tracking (U of U FOPEN Collaboration):** Export unnormalized MCLT correlation hyperstacks to train an Attention/SSM-based neural network. The goal is to replace the LMA fitter with a DNN regression head `(X, Y, vX, vY)` that applies learned "soft masks" to 16x16 macroblocks, running inference via TorchScript/DJL inside the Java pipeline. See `03_UU_RongRong_Hybrid_DNN_Architecture.md` for details.
- *Note: These deeper algorithmic optimizations are intentionally deferred. The strategy is to establish a working baseline first, expose the necessary low-bandwidth tile metrics via the MCP server, and then allow AI agents (Codex, Claude, Gemini) to autonomously sweep, analyze, and optimize these specific sub-problems.* - *Note: These deeper algorithmic optimizations are intentionally deferred. The strategy is to establish a working baseline first, expose the necessary low-bandwidth tile metrics via the MCP server, and then allow AI agents (Codex, Claude, Gemini) to autonomously sweep, analyze, and optimize these specific sub-problems.*
3. **FPGA / Hardware Teaming Roadmap (U of U Collaboration):** 3. **FPGA / Hardware Teaming Roadmap (U of U Collaboration):**
- **MCP for GTKWave:** Develop a Model Context Protocol (MCP) bridge to allow LLMs to natively analyze `.vcd` files. This will enable natural language querying of simulation waveform data (e.g., "Find the memory arbiter hang"). - **MCP for GTKWave:** Develop a Model Context Protocol (MCP) bridge to allow LLMs to natively analyze `.vcd` files. This will enable natural language querying of simulation waveform data (e.g., "Find the memory arbiter hang").
......
src/main/java/com/elphel/imagej/cuas/CuasMotion.java
View file @ a53575f5
...@@ -1066,7 +1066,7 @@ public class CuasMotion { ...@@ -1066,7 +1066,7 @@ public class CuasMotion {
return vf; return vf;
} }
public static void targetsAnalyze() { // not yet used public static void targetsAnalyze() {
int mode = 0; int mode = 0;
double par_val = 0.0; double par_val = 0.0;
int max_targets = 50; int max_targets = 50;
...@@ -1099,7 +1099,8 @@ public class CuasMotion { ...@@ -1099,7 +1099,8 @@ public class CuasMotion {
int num_pars = CuasMotionLMA.RSLT_LEN; // pvf_top_titles[0].length; // .RSLT_LEN; int num_pars = CuasMotionLMA.RSLT_LEN; // pvf_top_titles[0].length; // .RSLT_LEN;
int npar = CuasMotionLMA.RSLT_A; int npar = CuasMotionLMA.RSLT_A;
int fw = 9; int fw = 9;
String [] extra_choices = {"pX/pY"}; final int PAR_UPXY=-3,PAR_VPXY=-2,PAR_PXY=-1;
String [] extra_choices = {"upX/upY/range", "vX/vY/conf", "pX/pY"};
String [] choices = new String [num_pars+extra_choices.length]; String [] choices = new String [num_pars+extra_choices.length];
for (int i = -extra_choices.length; i < 0; i++) { for (int i = -extra_choices.length; i < 0; i++) {
choices[i + extra_choices.length] = String.format("%2d: %s", i, extra_choices[i+extra_choices.length]); // pvf_top_titles[0]) choices[i + extra_choices.length] = String.format("%2d: %s", i, extra_choices[i+extra_choices.length]); // pvf_top_titles[0])
...@@ -1109,7 +1110,7 @@ public class CuasMotion { ...@@ -1109,7 +1110,7 @@ public class CuasMotion {
} }
while (true) { while (true) {
GenericJTabbedDialog gd1 = new GenericJTabbedDialog("Select Range",600,300); GenericJTabbedDialog gd1 = new GenericJTabbedDialog("Select Range",600,500);
gd1.addNumericField("tile X (0.."+(wh[0] -1)+")", tx, 0, 3, "", gd1.addNumericField("tile X (0.."+(wh[0] -1)+")", tx, 0, 3, "",
"Select tile X"); "Select tile X");
gd1.addNumericField("tile Y (0.."+(wh[1] -1)+")", ty, 0, 3, "", gd1.addNumericField("tile Y (0.."+(wh[1] -1)+")", ty, 0, 3, "",
...@@ -1166,11 +1167,25 @@ public class CuasMotion { ...@@ -1166,11 +1167,25 @@ public class CuasMotion {
if (has_targets) { if (has_targets) {
int num_targ = multi_targets[nseq][ntile].length; int num_targ = multi_targets[nseq][ntile].length;
for (int ntarg = 0; ntarg < num_targ; ntarg++) { for (int ntarg = 0; ntarg < num_targ; ntarg++) {
if (npar == -1) { switch (npar) {
double px = tileSize + tx + tileSize/2 + multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_X]; case PAR_UPXY:
double py = tileSize + ty + tileSize/2 + multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_Y]; double upx = multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_FL_PX];
double upy = multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_FL_PY];
double urng = multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_FL_RANGE];
System.out.print(String.format("(%8.3f, %8.3f, %8.3f)",upx,upy,urng));
break;
case PAR_VPXY:
double vx = multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_VX];
double vy = multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_VY];
double vconf = multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_VSTR];
System.out.print(String.format("(%8.4f, %8.4f, %8.5f)",vx,vy,vconf));
break;
case PAR_PXY:
double px = tileSize * tx + tileSize/2 + multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_X];
double py = tileSize * ty + tileSize/2 + multi_targets[nseq][ntile][ntarg][CuasMotionLMA.RSLT_Y];
System.out.print(String.format("(%8.3f, %8.3f)",px,py)); System.out.print(String.format("(%8.3f, %8.3f)",px,py));
} else { break;
default:
System.out.print(String.format("%"+fw+"f",multi_targets[nseq][ntile][ntarg][npar])); System.out.print(String.format("%"+fw+"f",multi_targets[nseq][ntile][ntarg][npar]));
} }
if (ntarg < (num_targ-1)) { if (ntarg < (num_targ-1)) {
...@@ -1516,7 +1531,7 @@ public class CuasMotion { ...@@ -1516,7 +1531,7 @@ public class CuasMotion {
path); //String path) path); //String path)
} }
public static double [][][] getTargetsFromHyperAugment( public static double [][][] getTargetsFromHyperAugment_2025(
String [][] pvf_top_titles, String [][] pvf_top_titles,
String [][] pvf_titles, String [][] pvf_titles,
String path){ // add empty fields to the end of each target if shorter than CuasMotionLMA.RSLT_LEN String path){ // add empty fields to the end of each target if shorter than CuasMotionLMA.RSLT_LEN
...@@ -1560,6 +1575,59 @@ public class CuasMotion { ...@@ -1560,6 +1575,59 @@ public class CuasMotion {
return target_sequence; return target_sequence;
} }
public static double [][][] getTargetsFromHyperAugment(
String [][] pvf_top_titles,
String [][] pvf_titles,
String path){ // add empty fields to the end of each target if shorter than CuasMotionLMA.RSLT_LEN
int [] wh = new int [2];
if (pvf_top_titles == null) pvf_top_titles = new String[1][];
if (pvf_titles == null) pvf_titles = new String[1][];
double [][][] targets_file = ShowDoubleFloatArrays.readDoubleHyperstack(
path, // String path,
wh, // int [] wh, // should be null or int[2]
pvf_top_titles, // String [][] ptop_titles, // should be null or String [1][]
pvf_titles); // String [][] pslice_titles){// should be null or String [1][]
if (targets_file == null) {
return null;
}
int num_fields = targets_file.length;
// fix - discarding last slice if it is called CuasMotionLMA.EXTRA_SLICE_DISCARD_ON_LOAD ("Targets")
if (pvf_top_titles[0][pvf_top_titles[0].length-1].equals(CuasMotionLMA.EXTRA_SLICE_DISCARD_ON_LOAD)) {
System.out.println("getTargetsFromHyperAugment(): removing last slice called "+ pvf_top_titles[0][pvf_top_titles[0].length-1]);
num_fields--;
String [] ss = new String[num_fields];
System.arraycopy(pvf_top_titles[0], 0, ss, 0, num_fields);
pvf_top_titles[0] = ss;
}
final int num_fields_augmented = Math.max(num_fields, CuasMotionLMA.RSLT_LEN);
final int num_seq = targets_file[0].length;
final int num_tiles = targets_file[0][0].length;
final double [][][] target_sequence = new double [num_seq][num_tiles][];
final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0);
for (int ithread = 0; ithread < threads.length; ithread++) {
threads[ithread] = new Thread() {
public void run() {
double [] fields = new double[num_fields_augmented];
for (int nSeq = ai.getAndIncrement(); nSeq < num_seq; nSeq = ai.getAndIncrement()) {
for (int ntile = 0; ntile < num_tiles; ntile++) {
boolean has_data = false;
for (int npar = 0; npar < num_fields_augmented; npar++) {
fields[npar] = targets_file[npar][nSeq][ntile];
has_data |= !Double.isNaN(fields[npar]);
}
if (has_data) { // at least on is not NaN - create target
target_sequence[nSeq][ntile] = fields.clone();
}
}
}
}
};
}
ImageDtt.startAndJoin(threads);
return target_sequence;
}
...@@ -1875,10 +1943,9 @@ public class CuasMotion { ...@@ -1875,10 +1943,9 @@ public class CuasMotion {
String title, String title,
double [][][] corr2d, // null or [(fpixels.length - seq_length - start_frame) / corr_step)[][] double [][][] corr2d, // null or [(fpixels.length - seq_length - start_frame) / corr_step)[][]
int debugLevel){ int debugLevel){
// int scale_pairs = 1; // temporarily for testing boolean boost_offset = false; // true;
// if ((slow_scale_samples >= 1) && (slow_scale_offset <= 1)) { int corr_offset_boosted = boost_offset ? 2* ((int) Math.round(0.5 * corr_offset * boost_pairs)) : corr_offset;
// scale_pairs = 4; double scale_vxy = 1.0 / (boost_offset ? boost_pairs : 1.0);
// }
int seq_length = corr_offset + corr_pairs; int seq_length = corr_offset + corr_pairs;
int num_scenes = fpixels.length; int num_scenes = fpixels.length;
int num_corr_samples = (num_scenes - seq_length - start_frame) / corr_inc; int num_corr_samples = (num_scenes - seq_length - start_frame) / corr_inc;
...@@ -1892,19 +1959,21 @@ public class CuasMotion { ...@@ -1892,19 +1959,21 @@ public class CuasMotion {
if (boost_pairs > 1) { if (boost_pairs > 1) {
frame0 -= (int) Math.round((corr_pairs * (boost_pairs - 1))/ 2); frame0 -= (int) Math.round((corr_pairs * (boost_pairs - 1))/ 2);
} }
int frame1 = frame0 + corr_offset; int frame_cent = frame0 + corr_offset/2;
// int frame_cent = frame0 + seq_length/2; // debug only if (boost_offset) {
int frame_cent = frame0 + corr_offset/2; // debug only frame0 = frame_cent - corr_offset_boosted/2;
String suffix_param = "-"+frame_cent+"-"+corr_offset+"-"+corr_pairs; }
int frame1 = frame0 + corr_offset_boosted; // corr_offset;
String suffix_param = "-"+frame_cent+"-"+corr_offset_boosted+"-"+corr_pairs;
String dbg_suffix = (title != null) ? (title+suffix_param) : null; String dbg_suffix = (title != null) ? (title+suffix_param) : null;
if ((slow_scale_samples > 1) || (slow_scale_offset > 1)) { if ((slow_scale_samples > 1) || (slow_scale_offset > 1)) {
int frame_cent2 = 2 * frame0 + corr_offset; int frame_cent2 = 2 * frame0 + corr_offset_boosted;
int scan_center2 = frame_cent2 + corr_pairs; // twice index of scan center int scan_center2 = frame_cent2 + corr_pairs; // twice index of scan center
// int num_steps = corr_pairs / corr_ra_step; // total number of correlation pairs
corr_pairs_scaled = corr_pairs * slow_scale_samples; corr_pairs_scaled = corr_pairs * slow_scale_samples;
int frame_cent2_scaled = scan_center2 - corr_pairs_scaled; // * slow_scale_samples; int frame_cent2_scaled = scan_center2 - corr_pairs_scaled; // * slow_scale_samples;
frame0 = (frame_cent2_scaled - corr_offset * slow_scale_offset)/2; frame0 = (frame_cent2_scaled - corr_offset_boosted * slow_scale_offset)/2;
frame1 = frame0 + corr_offset * slow_scale_offset; frame1 = frame0 + corr_offset_boosted * slow_scale_offset;
frame_cent = (frame0 + frame1)/2; // debug only frame_cent = (frame0 + frame1)/2; // debug only
} }
...@@ -1935,13 +2004,14 @@ public class CuasMotion { ...@@ -1935,13 +2004,14 @@ public class CuasMotion {
cent_radius, // final double centroid_radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad) cent_radius, // final double centroid_radius, // 0 - all same weight, > 0 cosine(PI/2*sqrt(dx^2+dy^2)/rad)
n_recenter, // final int n_recenter); // re-center window around new maximum. 0 -no refines (single-pass) n_recenter, // final int n_recenter); // re-center window around new maximum. 0 -no refines (single-pass)
true); // final boolean calc_fraction ){ // calculate fraction inside center circle true); // final boolean calc_fraction ){ // calculate fraction inside center circle
for (int ntile = 0; ntile < num_tiles; ntile++) if (vector_field[ntile] != null){ for (int ntile = 0; ntile < num_tiles; ntile++) if (vector_field[ntile] != null){
double [] target = CuasMotionLMA.getEmpty(); double [] target = CuasMotionLMA.getEmpty();
// Store the raw displacement over one correlation-pair separation // Store the raw displacement over one correlation-pair separation
// (cuas_corr_offset scenes). Convert later with getFrameVelocityScale() // (cuas_corr_offset scenes). Convert later with getFrameVelocityScale()
// or getKeyFrameVelocityScale() depending on the consumer. // or getKeyFrameVelocityScale() depending on the consumer.
target[CuasMotionLMA.RSLT_VX] = vector_field[ntile][INDX_VX]; target[CuasMotionLMA.RSLT_VX] = vector_field[ntile][INDX_VX] * scale_vxy;
target[CuasMotionLMA.RSLT_VY] = vector_field[ntile][INDX_VY]; target[CuasMotionLMA.RSLT_VY] = vector_field[ntile][INDX_VY] * scale_vxy;
target[CuasMotionLMA.RSLT_VSTR] = vector_field[ntile][INDX_STRENGTH]; target[CuasMotionLMA.RSLT_VSTR] = vector_field[ntile][INDX_STRENGTH];
target[CuasMotionLMA.RSLT_VFRAC] = vector_field[ntile][INDX_FRAC]; target[CuasMotionLMA.RSLT_VFRAC] = vector_field[ntile][INDX_FRAC];
targets[nscan][ntile] = target; targets[nscan][ntile] = target;
...@@ -7088,7 +7158,7 @@ public class CuasMotion { ...@@ -7088,7 +7158,7 @@ public class CuasMotion {
ImagePlus imp_initial_boosted = showTargetSequence( ImagePlus imp_initial_boosted = showTargetSequence(
target_sequence_boost, // double [][][] vector_fields_sequence, target_sequence_boost, // double [][][] vector_fields_sequence,
slice_titles, // String [] titles, // all slices*frames titles or just slice titles or null slice_titles, // String [] titles, // all slices*frames titles or just slice titles or null
model_prefix+"-INITIAL-MOTION-BOOSTED",// String title, model_prefix+"-INITIAL-MOTION-BOOSTED"+boost_pairs,// String title,
!batch_mode, // boolean show, !batch_mode, // boolean show,
cuasMotion.tilesX); // int tilesX) { cuasMotion.tilesX); // int tilesX) {
parentCLT.saveImagePlusInModelDirectory(imp_initial_boosted); parentCLT.saveImagePlusInModelDirectory(imp_initial_boosted);
...@@ -7114,7 +7184,7 @@ public class CuasMotion { ...@@ -7114,7 +7184,7 @@ public class CuasMotion {
dbg_2d_corrs_boosted, // double[][] pixels, dbg_2d_corrs_boosted, // double[][] pixels,
cuasMotion.tilesX * (corr_size + 1), cuasMotion.tilesX * (corr_size + 1),
cuasMotion.tilesY * (corr_size + 1), cuasMotion.tilesY * (corr_size + 1),
model_prefix+"-CORR2D-BOOSTED", // -OFFS"+corr_offset+"-PAIRS"+corr_pairs, // String title, model_prefix+"-CORR2D-BOOSTED"+boost_pairs, // -OFFS"+corr_offset+"-PAIRS"+corr_pairs, // String title,
slice_titles); slice_titles);
if (!batch_mode) { if (!batch_mode) {
imp_corr2d.show(); imp_corr2d.show();
...@@ -8114,7 +8184,7 @@ public class CuasMotion { ...@@ -8114,7 +8184,7 @@ public class CuasMotion {
final int [][] filter5 = new int [num_seq][num_tiles]; final int [][] filter5 = new int [num_seq][num_tiles];
final Thread[] threads = ImageDtt.newThreadArray(); final Thread[] threads = ImageDtt.newThreadArray();
final AtomicInteger ai = new AtomicInteger(0); final AtomicInteger ai = new AtomicInteger(0);
final int dbg_tile = debug_now ?(12+23*80) : -1; final int dbg_tile = debug_now ?(53+38*80) : -1;
final int dbg_seq0 = debug_now ? 47: -1; final int dbg_seq0 = debug_now ? 47: -1;
final int dbg_seq1 = debug_now ? 49: -1; final int dbg_seq1 = debug_now ? 49: -1;
final int ihorizon = (lma_horizon > 0) ?( (int) Math.ceil(lma_horizon/GPUTileProcessor.DTT_SIZE)) : -1; // tiles with tileY >= ihorizon are removed final int ihorizon = (lma_horizon > 0) ?( (int) Math.ceil(lma_horizon/GPUTileProcessor.DTT_SIZE)) : -1; // tiles with tileY >= ihorizon are removed
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment