Commit a44f6c06 authored by Andrey Filippov's avatar Andrey Filippov

Merge branch 'foliage-gpu' of git.elphel.com:Elphel/imagej-elphel into foliage-gpu

parents 932fe03f c9acd835
......@@ -17,6 +17,7 @@ __pycache__
CLAUDE.md
GEMINI.md
MEMORY.md
ANDREY_CONTINUE.md
.claude/
.gemini/
*.orig
......@@ -40,6 +40,12 @@
</dependencyManagement>
<dependencies>
<!-- JNA: call the native CUDA tile-processor shim (libtileproc.so) for the JCuda->JNA GPU-layer migration. By Claude on 2026-06-25 -->
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>5.14.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
<dependency>
<groupId>com.google.code.gson</groupId>
......@@ -314,7 +320,41 @@
-->
</dependencies>
</profile>
</profiles>
<!-- Opt-in (mvn -Plibtorch ...): unpack the libtorch native runtime (cu128, ~3.8GB zip) from the
Elphel mirror into target/libtorch-dist for the native DNN backend (libtpdnn.so / CuasDnnLocal).
OFF by default so the normal build never downloads it. Publish the artifact to the mirror with
tile_processor_gpu/jna/publish_libtorch_mirror.sh. By Claude on 06/27/2026. -->
<profile>
<id>libtorch</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.1.2</version>
<executions>
<execution>
<id>unpack-libtorch</id>
<phase>generate-resources</phase>
<goals><goal>unpack</goal></goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.pytorch</groupId>
<artifactId>libtorch-cxx11-cu128</artifactId>
<version>2.7.1</version>
<type>zip</type>
<outputDirectory>${project.build.directory}/libtorch-dist</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
<resources>
......
#!/usr/bin/env python3
# By Claude on 07/04/2026
"""Migrate corr-xml configuration files to the current curt_* key format.
CuasRtParameters history:
pre-07/2026 keys CLT_PARAMETERS._imp_curt_<name> (curt_* fields inside IntersceneMatchParameters,
then the extracted class still saved under imp with the same keys)
07/2026 - keys CLT_PARAMETERS._curt_<name> (CuasRtParameters is a CLTParameters peer,
like imp/ofp/ilp)
The Java reader keeps a legacy fallback (it reads _imp_curt_* when present, then
_curt_* overrides), so unmigrated files still load. This script rewrites the keys
so files are in the current format and future removal of the fallback is safe.
Usage:
migrate_curt_config.py FILE [FILE ...] # in-place, keeps FILE.bak
migrate_curt_config.py --dry-run FILE ... # report only, no changes
migrate_curt_config.py --no-backup FILE ... # in-place without .bak
Only the literal substring '_imp_curt_' inside key="..." attributes is renamed to
'_curt_'; everything else (values, formatting, comments) is byte-preserved. If a
file already contains a _curt_ key that a rename would duplicate, the old key is
dropped (the new one wins) and a warning is printed.
A provenance XML comment (absolute path of this script + timestamp) is inserted
right after the <properties> line of every modified file. Java's XML properties
loader ignores XML comments, but note it also DROPS them if the application
re-saves the file - the marker survives only until the next save from ImageJ.
"""
import argparse
import datetime
import os
import re
import shutil
import sys
KEY_RE = re.compile(r'(<entry key=")([^"]*)_imp_curt_([^"]*)(")')
def migrate_text(text):
"""Return (new_text, n_renamed, dropped_keys)."""
existing_new = set(re.findall(r'<entry key="([^"]*_curt_[^"]*)"', text))
existing_new = {k for k in existing_new if '_imp_curt_' not in k}
renamed = [0]
dropped = []
out_lines = []
for line in text.splitlines(keepends=True):
m = KEY_RE.search(line)
if m:
new_key = m.group(2) + '_curt_' + m.group(3)
if new_key in existing_new:
dropped.append(m.group(2) + '_imp_curt_' + m.group(3))
continue # drop the whole legacy entry line - the new key already exists
line = KEY_RE.sub(r'\g<1>\g<2>_curt_\g<3>\g<4>', line)
renamed[0] += 1
out_lines.append(line)
return ''.join(out_lines), renamed[0], dropped
def add_provenance_comment(text):
"""Insert a provenance XML comment after the <properties> line."""
stamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
marker = ('<!-- curt_* keys migrated (_imp_curt_* to _curt_*) by %s on %s -->\n'
% (os.path.abspath(__file__), stamp))
lines = text.splitlines(keepends=True)
for i, line in enumerate(lines):
if '<properties>' in line:
lines.insert(i + 1, marker)
return ''.join(lines)
return marker + text # no <properties> line found - prepend (still valid for grep)
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('files', nargs='+', help='corr-xml files to migrate')
ap.add_argument('--dry-run', action='store_true', help='report only, do not modify')
ap.add_argument('--no-backup', action='store_true', help='do not keep a .bak copy')
args = ap.parse_args()
status = 0
for path in args.files:
try:
with open(path, encoding='utf-8') as f:
text = f.read()
except OSError as e:
print(f"{path}: ERROR - {e}", file=sys.stderr)
status = 1
continue
new_text, n, dropped = migrate_text(text)
for k in dropped:
print(f"{path}: WARNING - dropped legacy entry '{k}' (current-format key already present)")
if n == 0 and not dropped:
print(f"{path}: already current (no _imp_curt_ keys)")
continue
if args.dry_run:
print(f"{path}: would rename {n} key(s)" +
(f", drop {len(dropped)} duplicate legacy entrie(s)" if dropped else ""))
continue
if not args.no_backup:
shutil.copy2(path, path + '.bak')
with open(path, 'w', encoding='utf-8') as f:
f.write(add_provenance_comment(new_text))
print(f"{path}: renamed {n} key(s)" +
(f", dropped {len(dropped)} duplicate legacy entrie(s)" if dropped else "") +
("" if args.no_backup else f" (backup: {path}.bak)"))
return status
if __name__ == '__main__':
sys.exit(main())
......@@ -40,6 +40,7 @@ import com.elphel.imagej.common.WindowTools;
import com.elphel.imagej.correction.CorrectionColorProc;
import com.elphel.imagej.lwir.LwirReaderParameters;
import com.elphel.imagej.tileprocessor.BiQuadParameters;
import com.elphel.imagej.tileprocessor.CuasRtParameters; // By Claude on 07/04/2026
import com.elphel.imagej.tileprocessor.ImageDtt;
import com.elphel.imagej.tileprocessor.ImageDttParameters;
import com.elphel.imagej.tileprocessor.IntersceneGlobalLmaParameters;
......@@ -75,6 +76,7 @@ public class CLTParameters {
private double scale_strength_main = 1.0;// leave as is
private double scale_strength_aux = 0.3;// reduce to match lower sigma
public boolean norm_kern = true; // normalize kernels
public boolean norm_kern_abs = false;// normalize kernels absolute value (for those with sum=0
public boolean gain_equalize = false;// equalize green channel gain (bug fix for wrong exposure in Exif ?)
public boolean colors_equalize = true; // equalize R/G, B/G of the individual channels
public boolean nosat_equalize = true; // Skip saturated when adjusting gains
......@@ -1183,6 +1185,7 @@ public class CLTParameters {
public IntersceneGlobalLmaParameters iglp = new IntersceneGlobalLmaParameters();
public InterNoiseParameters inp = new InterNoiseParameters();
public LWIRWorldParameters lwp = new LWIRWorldParameters();
public CuasRtParameters curt = new CuasRtParameters(); // CUAS real-time (was imp.curt) // By Claude on 07/04/2026
public HashMap<String,Double> z_corr_map = new HashMap<String,Double>(); //old one
public HashMap<String,Double> infinity_distace_map = new HashMap<String,Double>(); //new one
......@@ -1317,6 +1320,7 @@ public class CLTParameters {
properties.setProperty(prefix+"scale_strength_aux", this.scale_strength_aux+"");
properties.setProperty(prefix+"norm_kern", this.norm_kern+"");
properties.setProperty(prefix+"norm_kern_abs", this.norm_kern_abs+"");
properties.setProperty(prefix+"gain_equalize", this.gain_equalize+"");
properties.setProperty(prefix+"colors_equalize", this.colors_equalize+"");
properties.setProperty(prefix+"nosat_equalize", this.nosat_equalize+"");
......@@ -2338,6 +2342,7 @@ public class CLTParameters {
iglp.setProperties (prefix+"_iglp_", properties);
inp.setProperties (prefix+"_inp_", properties);
lwp.setProperties (prefix+"_lwp_", properties);
curt.setProperties (prefix+"_curt_", properties); // By Claude on 07/04/2026
}
......@@ -2374,6 +2379,7 @@ public class CLTParameters {
if (properties.getProperty(prefix+"scale_strength_aux")!=null) this.scale_strength_aux=Double.parseDouble(properties.getProperty(prefix+"scale_strength_aux"));
if (properties.getProperty(prefix+"norm_kern")!=null) this.norm_kern=Boolean.parseBoolean(properties.getProperty(prefix+"norm_kern"));
if (properties.getProperty(prefix+"norm_kern_abs")!=null) this.norm_kern_abs=Boolean.parseBoolean(properties.getProperty(prefix+"norm_kern_abs"));
if (properties.getProperty(prefix+"gain_equalize")!=null) this.gain_equalize=Boolean.parseBoolean(properties.getProperty(prefix+"gain_equalize"));
if (properties.getProperty(prefix+"colors_equalize")!=null) this.colors_equalize=Boolean.parseBoolean(properties.getProperty(prefix+"colors_equalize"));
if (properties.getProperty(prefix+"nosat_equalize")!=null) this.nosat_equalize=Boolean.parseBoolean(properties.getProperty(prefix+"nosat_equalize"));
......@@ -3408,6 +3414,8 @@ public class CLTParameters {
iglp.getProperties (prefix+"_iglp_", properties);
inp.getProperties (prefix+"_inp_", properties);
lwp.getProperties (prefix+"_lwp_", properties);
curt.getProperties (prefix+"_imp_curt_", properties); // legacy keys (curt was inside imp until 07/2026) // By Claude on 07/04/2026
curt.getProperties (prefix+"_curt_", properties); // current keys override legacy if both present // By Claude on 07/04/2026
}
public boolean showJDialog() {
......@@ -3456,7 +3464,8 @@ public class CLTParameters {
gd.addNumericField("Scale all correlation strengths (to compensate correlation sigma) for aux camera", this.scale_strength_aux, 4);
gd.addCheckbox ("Normalize kernels", this.norm_kern);
gd.addCheckbox ("Equalize green channel gain of the individual cnannels (bug fix for exposure)", this.gain_equalize);
gd.addCheckbox ("Normalize kernels absolute values", this.norm_kern_abs);
gd.addCheckbox ("Equalize green channel gain of the individual cnannels (bug fix for exposure)", this.gain_equalize);
gd.addCheckbox ("Equalize R/G, B/G balance of the individual channels", this.colors_equalize);
gd.addCheckbox ("Skip saturated when adjusting gains", this.nosat_equalize);
gd.addNumericField("Saturation level of the most saturated color channel", this.sat_level, 4);
......@@ -4898,6 +4907,9 @@ public class CLTParameters {
this.imp.dialogQuestions(gd);
gd.addTab ("CUAS RT", "CUAS Real Time");
this.curt.dialogQuestions(gd); // By Claude on 07/04/2026
this.lwp.dialogQuestions(gd);
gd.addTab ("Inter-LMA", "parameters for the interscene LMA fitting");
......@@ -4992,6 +5004,7 @@ public class CLTParameters {
this.scale_strength_main = gd.getNextNumber();
this.scale_strength_aux = gd.getNextNumber();
this.norm_kern= gd.getNextBoolean();
this.norm_kern_abs= gd.getNextBoolean();
this.gain_equalize= gd.getNextBoolean();
this.colors_equalize= gd.getNextBoolean();
this.nosat_equalize= gd.getNextBoolean();
......@@ -5965,6 +5978,7 @@ public class CLTParameters {
this.lwir.dialogAnswers(gd);
this.ofp.dialogAnswers(gd);
this.imp.dialogAnswers(gd);
this.curt.dialogAnswers(gd); // By Claude on 07/04/2026
this.lwp.dialogAnswers(gd);
this.ilp.dialogAnswers(gd);
this.iglp.dialogAnswers(gd);
......
......@@ -68,7 +68,9 @@ public class EyesisCorrectionParameters {
"resultsDirectory", // 6
"cuasSeed", // 7
"uasLogs", // 8
"skyMask"}; // 9
"skyMask", // 9
"cuasSynth", // 10 shared synthetic-grid dir (curt_synth_src), valid for all sequences. By Claude on 06/24/2026
"cuasNoise"}; // 11 INLINE per-level L2 noise-scale numbers (NOT a path); empty -> sqrt default. By Claude on 06/24/2026
public static final int KEY_INDEX_ROOT_DIRECTORY = 0;
public static final int KEY_INDEX_SOURCE_DIRECTORY = 1;
public static final int KEY_INDEX_LINKED_MODELS = 2;
......@@ -79,6 +81,8 @@ public class EyesisCorrectionParameters {
public static final int KEY_INDEX_CUAS_SEED = 7;
public static final int KEY_INDEX_UAS_LOGS = 8;
public static final int KEY_INDEX_SKY_MASK = 9;
public static final int KEY_INDEX_CUAS_SYNTH = 10; // By Claude on 06/24/2026
public static final int KEY_INDEX_CUAS_NOISE = 11; // inline per-level noise scales. By Claude on 06/24/2026
public static final String AUX_PREFIX = "AUX-";
public boolean swapSubchannels01= true; // false; // (false: 0-1-2, true - 1-0-2)
......@@ -129,6 +133,8 @@ public class EyesisCorrectionParameters {
public String cuasSeedDir= "";
public boolean useCuasSeedDir= false;
public String cuasSkyMask = ""; // TIFF image 640x512 where 1.0 - sky, 0.0 - ground, blurred with GB (now sigma==2.0)
public String cuasSynth = ""; // shared dir holding the synthetic-grid TIFF (curt_synth_src) - valid for ALL sequences, resolved from the list SET key "cuasSynth" (relative to rootDirectory), like cuasSkyMask; empty -> per-sequence model dir (old behavior). By Claude on 06/24/2026
public String cuasNoise = ""; // INLINE per-level L2 noise-scale numbers from the list SET key "cuasNoise" (e.g. "0.354,0.5,0.707,1.0,1.414,2.0"); NOT a path. Empty -> theoretical sqrt(2)^(L-3) default (computed in CuasDetectRT). By Claude on 06/24/2026
public String cuasUasLogs = ""; // json file path containing UAS logs
public double cuasUasTimeStamp = 0.0; // timestamp corresponding to the UAS time 0.0
public double [] cuasCameraATR = {0, 0, 0};
......@@ -322,6 +328,8 @@ public class EyesisCorrectionParameters {
cp.useCuasSeedDir= this.useCuasSeedDir;
cp.cuasSkyMask = this.cuasSkyMask;
cp.cuasSynth = this.cuasSynth;
cp.cuasNoise = this.cuasNoise;
cp.cuasUasLogs = this.cuasUasLogs;
cp.cuasUasTimeStamp = this.cuasUasTimeStamp;
cp.cuasCameraATR = this.cuasCameraATR.clone();
......@@ -1833,7 +1841,7 @@ public class EyesisCorrectionParameters {
if (dir_map.get(KEY_DIRS[i]).length() > 0){
Path dir_path=base_path.resolve(Paths.get(dir_map.get(KEY_DIRS[i])));
File dir_file = new File(dir_path.toString());
if ((i != KEY_INDEX_UAS_LOGS) && (i != KEY_INDEX_SKY_MASK)) { // cuasUasLogs, cuasSkyMask are files, not directories
if ((i != KEY_INDEX_UAS_LOGS) && (i != KEY_INDEX_SKY_MASK) && (i != KEY_INDEX_CUAS_SYNTH) && (i != KEY_INDEX_CUAS_NOISE)) { // cuasUasLogs/cuasSkyMask=files; cuasSynth=input dir; cuasNoise=inline numbers (not a path) - don't auto-create. By Claude on 06/24/2026
if (!dir_file.exists()) {
if (MKDIRS_ALLOW) {
dir_file.mkdirs();
......@@ -1901,7 +1909,20 @@ public class EyesisCorrectionParameters {
case KEY_INDEX_SKY_MASK: // 9: // cuasSeed
this.cuasSkyMask = dir_string; // dir_path.toString();
System.out.println("this.cuasSkyMask=" + this.cuasSkyMask);
break;
case KEY_INDEX_CUAS_SYNTH: // 10: shared synthetic-grid dir (curt_synth_src), all sequences. By Claude on 06/24/2026
this.cuasSynth = dir_string;
System.out.println("this.cuasSynth=" + this.cuasSynth);
break;
case KEY_INDEX_CUAS_NOISE: // 11: INLINE per-level noise scales (numbers, NOT a path). By Claude on 06/24/2026
{
StringBuilder nsb = new StringBuilder(dir_map.get(KEY_DIRS[i])); // first number
ArrayList<String> nx = extra_map.get(KEY_DIRS[i]); // remaining numbers
if (nx != null) for (String v : nx) nsb.append(",").append(v);
this.cuasNoise = nsb.toString();
}
System.out.println("this.cuasNoise=" + this.cuasNoise);
break;
}
}
......
......@@ -315,7 +315,13 @@ public class GenericJTabbedDialog implements ActionListener {
label.setHorizontalAlignment(JLabel.RIGHT);
tab.add(label,gbc);
gbc.gridx = 1;
tab.add(component,gbc);
if (component instanceof JCheckBox) { // don't stretch checkboxes - keep the click target at the box, not the whole half-width cell (a near-scrollbar click was toggling them) // By Claude on 06/20/2026
gbc.fill = GridBagConstraints.NONE; gbc.anchor = GridBagConstraints.WEST;
tab.add(component,gbc);
gbc.fill = GridBagConstraints.HORIZONTAL; gbc.anchor = GridBagConstraints.CENTER; // restore for the following rows
} else {
tab.add(component,gbc);
}
}
}
// Add empty label that would push up all other rows
......
......@@ -38,6 +38,7 @@ public class GenericJTabbedDialogMcp extends GenericJTabbedDialog{
private static long dialogTimeoutMs = 30000L;
public boolean mcp_mode = false;
private boolean mcpCanceled = false;
private McpDialogSession mcpSession = null; // this dialog's OWN session: the registry is cleared on submit BEFORE the getNext*() reads run, so reads must not go through it // By Claude on 07/07/2026
private final List<McpDialogField> mcpFields = new ArrayList<McpDialogField>();
private final String dialogTitle;
private String currentTab = "";
......@@ -192,7 +193,8 @@ public class GenericJTabbedDialogMcp extends GenericJTabbedDialog{
if (mcp_mode) {
// codex 2026-01-25: publish dialog schema to MCP registry
readIndex = 0;
boolean applied = McpDialogRegistry.setCurrent(new McpDialogSession(dialogTitle, mcpFields));
mcpSession = new McpDialogSession(dialogTitle, mcpFields); // By Claude on 07/07/2026: keep own reference (see field comment)
boolean applied = McpDialogRegistry.setCurrent(mcpSession);
if (!applied) {
System.out.println("MCP: dialog already active, ignoring \"" + dialogTitle + "\"");
}
......@@ -218,7 +220,7 @@ public class GenericJTabbedDialogMcp extends GenericJTabbedDialog{
}
private String getValueOrDefault(McpDialogField field) {
McpDialogSession session = McpDialogRegistry.getCurrent();
McpDialogSession session = mcpSession; // By Claude on 07/07/2026: was McpDialogRegistry.getCurrent() - always null here (registry cleared on submit before the reads), so /mcp/dialog/values was silently discarded
if (session != null && field != null) {
String value = session.getValue(field.label);
if (value != null) {
......@@ -299,7 +301,7 @@ public class GenericJTabbedDialogMcp extends GenericJTabbedDialog{
public boolean showDialog() {
if (mcp_mode) {
buildDialog();
McpDialogSession session = McpDialogRegistry.getCurrent();
McpDialogSession session = mcpSession; // By Claude on 07/07/2026: own session (registry may hold another dialog / be cleared by submit)
if (session != null) {
long timeout = dialogTimeoutMs;
Boolean ok = (timeout <= 0) ? session.awaitSubmit(Long.MAX_VALUE) : session.awaitSubmit(timeout);
......@@ -309,7 +311,9 @@ public class GenericJTabbedDialogMcp extends GenericJTabbedDialog{
mcpCanceled = false;
}
}
McpDialogRegistry.setCurrent(null);
if (McpDialogRegistry.getCurrent() == mcpSession) { // By Claude on 07/07/2026: clear only OUR session (submit normally cleared it; this covers the timeout path without clobbering a newer dialog)
McpDialogRegistry.setCurrent(null);
}
return true;
}
else return super.showDialog();
......
......@@ -692,9 +692,80 @@ public class Cuas {
return center_CLT;
}
/**
* Rebuild the center IMAGE from the current in-memory scene image data (e.g. after
* subtracting a just-computed FPN) WITHOUT persisting or mutating anything: no
* .cuas / -CLT / DSI / IMS files are written and center_CLT.center_clt /
* center_CLT.image_center are left untouched. Always a NEW average - no parent
* cumulative is folded in (unlike stepCenterClt() which seeds from the current
* center and would keep its content residual). Used by the FPN second
* back-propagation iteration in CorrectionFPN.cuasSubtractFpn(), where the rebuilt
* center is only a temporary measurement reference.
* By Claude on 07/06/2026
* @param clt_parameters processing parameters
* @param quadCLTs scenes to accumulate; pass a slice covering an integer number of
* camera rotations so azimuth sectors are equally weighted
* @param center_CLT existing center (geometry/poses reference), must have center CLT
* @param disparity_center center-grid disparity (from backPrepareCenter()/getDisparityCenter())
* @param sensor_mask -1 - all
* @param debugLevel debug level
* @return image-domain center as convertCenterClt() returns, or null on failure
*/
public static double [][] rebuildCenterImageInMemory( // By Claude on 07/06/2026
CLTParameters clt_parameters,
QuadCLT [] quadCLTs,
QuadCLT center_CLT,
double [] disparity_center,
int sensor_mask, // -1 - all;
int debugLevel) {
if (!center_CLT.hasCenterClt()) {
System.out.println("rebuildCenterImageInMemory(): center_CLT does not have .center_clt data, bailing out");
return null;
}
double cuas_clt_variant = clt_parameters.imp.cuas_clt_variant; // 10;
double cuas_clt_threshold = clt_parameters.imp.cuas_clt_threshold; // 20;
double cuas_decay_average = clt_parameters.imp.cuas_decay_average; // 100;
double cuas_keep_fraction = clt_parameters.imp.cuas_keep_fraction; // 0.9;
double cuas_clt_decrease = clt_parameters.imp.cuas_clt_decrease; // 0.01;
double dts = center_CLT.getTimeStamp();
CuasData newCuasData = getTDComboSceneSequence(
clt_parameters, // CLTParameters clt_parameters,
null, // double [][] ref_pXpYD,
true, // boolean save_weights,
sensor_mask, // int sensor_mask,
null, // Rectangle fov_tiles,
OpticalFlow.ZERO3, // double [] stereo_xyz, // offset reference camera {x,y,z}
OpticalFlow.ZERO3, // double [] stereo_atr_in, // center already exists - zero ATR
disparity_center, // double [] ref_disparity,
quadCLTs, // QuadCLT [] quadCLTs,
center_CLT, // QuadCLT refCLT,
null, // CuasData cuasData - no parent cumulative: new average only
true, // final boolean clt_create, // create new variants
cuas_clt_variant, // final double clt_threshold,
cuas_decay_average, // final double clt_decay,
cuas_clt_decrease, // final double clt_decrease,
debugLevel); // int debugLevel)
if (newCuasData == null) {
System.out.println("rebuildCenterImageInMemory(): getTDComboSceneSequence() failed");
return null;
}
if (cuas_keep_fraction < 1.0) { // same filtering as createCenterClt() applies before collapse
newCuasData = filter(
newCuasData, // final CuasData cuasData,
cuas_decay_average, // final double cuas_decay, // seconds e times
cuas_keep_fraction, // final double keep_fraction,
debugLevel); // final int debugLevel)
}
float [] fclt = newCuasData.collapse(
cuas_clt_threshold, // final double tolerance, // NaN works as infinity
cuas_decay_average, // final double decay,
dts)[0]; // final double ts_now)
return center_CLT.convertCenterClt(new float [][] {fclt}); // does NOT touch center_CLT state
}
public static void fillNanDsi(
double [][] ref_pXpYD,
double [][] dsi,
......
/**
**
** CuasQC.java - per-product quality metrics with pass bands (regression protection)
**
** Copyright (C) 2026 Elphel, Inc.
**
** -----------------------------------------------------------------------------**
**
** CuasQC.java is free software: you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation, either version 3 of the License, or
** (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with this program. If not, see <http://www.gnu.org/licenses/>.
** -----------------------------------------------------------------------------**
**
*/
package com.elphel.imagej.cuas;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedHashMap;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* Quality-metrics harness (Andrey's design 07/07/2026, after the v013-ROWCOL-01 junk):
* once a pipeline step is "done and verified", detect regressions by per-product HEALTH
* STATISTICS with pass bands - NOT bit comparison (a photometric recalibration shifts
* values a little; these invariants survive it and transfer across image sequences).
* Bands are sequence-independent (defined here in code, in physical/calibrated units for
* this LWIR16 hardware class); per-sequence VALUES land in <center>-QC.json next to the
* products (append-per-run, with timestamps), so any run answers "was it healthy" and
* trends are visible. A FAIL prints loudly but never aborts the run.
*
* Increment 1 (07/07/2026): ROWCOL vectors + FPN. The ROWCOL bands would have caught both
* of today's defects (baseline-mix rms 83, edge-column p95 ~1000 vs legacy 8.5/338).
* By Claude on 07/07/2026, from Andrey's design.
*/
public class CuasQC {
public static final int QC_VERSION = 1;
public static final String SUFFIX = "-QC.json";
private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
// --- pass bands (sequence-independent, LWIR16 class; reference measurements in comments) ---
public static final double ROWCOL_MAX_RMS = 25.0; // legacy method 8.5; broken baseline-mix 83; broken edges 53
public static final double ROWCOL_MAX_SCENE_P95 = 500.0; // p95 of per-scene max |vector|; legacy 338 (own x=2 edge artifact included)
public static final double FPN_MAX_RMS = 150.0; // measured 40.5 (2026-07-07 vintage, rot 164.9)
public static final double FPN_MAX_LINE_ROW = 25.0; // FPN line-structure rms, per-row means; measured 6.4
public static final double FPN_MAX_LINE_COL = 25.0; // per-col means; measured 12.3
public static class Check {
public String name;
public double value;
public double max;
public boolean pass;
public Check(String name, double value, double max) {
this.name = name; this.value = value; this.max = max; this.pass = (value <= max);
}
}
public static class Entry {
public String saved; // timestamp of this QC evaluation
public String product; // "ROWCOL", "FPN", ...
public boolean calculated; // true - just calculated this run; false - reused from file
public boolean pass;
public LinkedHashMap<String, Double> metrics;
public ArrayList<Check> checks;
}
public static class QcFile {
public int qc_version = QC_VERSION;
public ArrayList<Entry> entries = new ArrayList<Entry>();
}
/** ROWCOL health: vector magnitudes must stay at the line-noise scale.
* @param row_col {rows[scene][sensor][width], cols[scene][sensor][height]} */
public static boolean checkRowCol(
String x3d_directory,
String center_name,
double [][][][] row_col,
boolean calculated) {
double sum2 = 0.0; long n = 0;
final int num_scenes = row_col[0].length;
final double [] scene_max = new double [num_scenes];
for (int f = 0; f < row_col.length; f++) {
for (int nscene = 0; nscene < num_scenes; nscene++) if (row_col[f][nscene] != null) {
for (double [] vec : row_col[f][nscene]) if (vec != null) {
for (double v : vec) if (!Double.isNaN(v)) {
sum2 += v*v; n++;
final double a = Math.abs(v);
if (a > scene_max[nscene]) scene_max[nscene] = a;
}
}
}
}
final double rms = Math.sqrt(sum2/Math.max(n,1));
final LinkedHashMap<String,Double> metrics = new LinkedHashMap<String,Double>();
metrics.put("rms", rms);
metrics.put("scene_max_med", percentile(scene_max, 50));
metrics.put("scene_max_p95", percentile(scene_max, 95));
metrics.put("scene_max_max", percentile(scene_max, 100));
final ArrayList<Check> checks = new ArrayList<Check>();
checks.add(new Check("rms", rms, ROWCOL_MAX_RMS));
checks.add(new Check("scene_max_p95", metrics.get("scene_max_p95"), ROWCOL_MAX_SCENE_P95));
return report(x3d_directory, center_name, "ROWCOL", calculated, metrics, checks);
}
/** FPN health: overall scale + row/col line-structure energy (per-sensor line means
* after removing each sensor's global mean, rms pooled over sensors). */
public static boolean checkFpn(
String x3d_directory,
String center_name,
double [][][] fpn,
int width,
boolean calculated) {
double sum2 = 0.0; long n = 0;
double line_row2 = 0.0, line_col2 = 0.0; long nrow = 0, ncol = 0;
for (int nsens = 0; nsens < fpn.length; nsens++) if ((fpn[nsens] != null) && (fpn[nsens][0] != null)) {
final double [] px = fpn[nsens][0];
final int height = px.length/width;
double mean = 0.0; long nm = 0;
for (double v : px) if (!Double.isNaN(v)) { mean += v; nm++; }
mean /= Math.max(nm,1);
final double [] rowm = new double [height];
final double [] coln = new double [width];
final double [] rown = new double [height];
final double [] colm = new double [width];
int idx = 0;
for (int y = 0; y < height; y++) for (int x = 0; x < width; x++) {
final double v = px[idx++];
if (Double.isNaN(v)) continue;
final double d = v - mean;
sum2 += d*d; n++;
rowm[y] += d; rown[y]++;
colm[x] += d; coln[x]++;
}
for (int y = 0; y < height; y++) if (rown[y] > 0) { final double d = rowm[y]/rown[y]; line_row2 += d*d; nrow++; }
for (int x = 0; x < width; x++) if (coln[x] > 0) { final double d = colm[x]/coln[x]; line_col2 += d*d; ncol++; }
}
final LinkedHashMap<String,Double> metrics = new LinkedHashMap<String,Double>();
metrics.put("rms", Math.sqrt(sum2/Math.max(n,1)));
metrics.put("line_row", Math.sqrt(line_row2/Math.max(nrow,1)));
metrics.put("line_col", Math.sqrt(line_col2/Math.max(ncol,1)));
final ArrayList<Check> checks = new ArrayList<Check>();
checks.add(new Check("rms", metrics.get("rms"), FPN_MAX_RMS));
checks.add(new Check("line_row", metrics.get("line_row"), FPN_MAX_LINE_ROW));
checks.add(new Check("line_col", metrics.get("line_col"), FPN_MAX_LINE_COL));
return report(x3d_directory, center_name, "FPN", calculated, metrics, checks);
}
private static double percentile(double [] a, double pct) {
final double [] s = a.clone();
Arrays.sort(s);
final int k = (int) Math.min(s.length-1, Math.round(0.01*pct*(s.length-1)));
return s[k];
}
/** Print the QC block, append the entry to <center>-QC.json, return overall pass. */
private static boolean report(
String x3d_directory, String center_name, String product, boolean calculated,
LinkedHashMap<String,Double> metrics, ArrayList<Check> checks) {
final Entry entry = new Entry();
entry.saved = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date());
entry.product = product;
entry.calculated = calculated;
entry.metrics = metrics;
entry.checks = checks;
entry.pass = true;
for (Check c : checks) entry.pass &= c.pass;
final StringBuilder sb = new StringBuilder();
sb.append("QC: ").append(product).append(calculated ? " (calculated)" : " (reused)")
.append(entry.pass ? " PASS" : " *** FAIL ***");
for (Check c : checks) if (!c.pass) {
sb.append(String.format(" [%s=%.2f > %.2f]", c.name, c.value, c.max));
}
sb.append(" metrics: ");
for (java.util.Map.Entry<String,Double> m : metrics.entrySet()) {
sb.append(String.format("%s=%.2f ", m.getKey(), m.getValue()));
}
System.out.println(sb.toString());
if (!entry.pass) {
System.out.println("QC: *** "+product+" is OUT OF ITS HEALTH BANDS - the product is suspect, "+
"inspect before trusting this run (bands in CuasQC.java) ***");
}
try {
final String path = x3d_directory + java.io.File.separator + center_name + SUFFIX;
QcFile qc = null;
if (Files.exists(Paths.get(path))) {
try (FileReader fr = new FileReader(path)) {
qc = GSON.fromJson(fr, QcFile.class);
} catch (Exception e) {
System.out.println("QC: could not parse existing "+path+" ("+e.getMessage()+") - starting a new file");
}
}
if ((qc == null) || (qc.entries == null)) qc = new QcFile();
qc.entries.add(entry);
try (FileWriter fw = new FileWriter(path)) {
GSON.toJson(qc, fw);
}
} catch (IOException e) {
System.out.println("QC: failed to save QC json: "+e.getMessage());
}
return entry.pass;
}
}
......@@ -75,6 +75,7 @@ public class CuasRanging {
}
public ImagePlus prepareFpixels(){
boolean dbg_save_fpixels = clt_parameters.curt.dbg_fpixels; // was hardcoded true: gates the EXTRA per-sensor -CUAS-INDIVIDUAL-CUAS-DBG render pass (~11GB) + the -CUAS-MERGED-CUAS-DBG save; -CUAS-RT-RENDER (rend_test) is the RT-chain full render // By Claude on 07/05/2026
double [][]combo_dsi = getCenter_CLT().comboFromMain();
double [][] dls = {
combo_dsi[OpticalFlow.COMBO_DSN_INDX_DISP], // **** null on second scene sequence
......@@ -135,6 +136,52 @@ public class CuasRanging {
double [] cuas_atr = OpticalFlow.ZERO3;
String scenes_suffix = getCenter_CLT().getImageName()+"-CUAS"; // "1747829900_781803-SEQ-FG-MONO-FPN";
boolean merge_all = false; // clt_parameters.imp.merge_all || !um_mono; // no unsharp mask -> terrain->merge_all
int sensor_mask = merge_all? 1 : -1;
if (dbg_save_fpixels) {
// By Claude on 07/02/2026: per-sensor averages can only be calculated from slices (the precalculated CLT center
// average is merged-channels only), so add_average implies calculating here. Number of average frames is
// variable: 0, 1 (full) or 2 (full + center fraction), matching the merged sequence convention.
boolean dbg_add_average = clt_parameters.imp.add_average || clt_parameters.imp.calculate_average; // By Claude on 07/02/2026
int [] dbg_average_range = (clt_parameters.imp.add_center_average && dbg_add_average)? new int[2] : null; // By Claude on 07/02/2026
if (dbg_average_range != null) { // By Claude on 07/02/2026: same as in renderSceneSequence() wrapper
int num_scenes = scenes.length;
dbg_average_range[0] = (int) Math.round (num_scenes * (1 - clt_parameters.imp.center_avg_frac)/2);
dbg_average_range[0] = Math.max(0, dbg_average_range[0]);
dbg_average_range[0] = Math.min(num_scenes-1, dbg_average_range[0]);
dbg_average_range[1] = num_scenes-1 - dbg_average_range[0];
dbg_average_range[1] = Math.max(dbg_average_range[0], dbg_average_range[1]);
dbg_average_range[1] = Math.min(num_scenes-1, dbg_average_range[1]);
}
ImagePlus imp_targets= OpticalFlow.renderSceneSequence(
clt_parameters, // CLTParameters clt_parameters,
true, // center_CLT.hasCenterClt(), // boolean mode_cuas,
false, // clt_parameters.imp.um_mono, // boolean um_mono,
dbg_add_average, // boolean insert_average, // By Claude on 07/02/2026: was clt_parameters.imp.calculate_average
dbg_average_range, // int [] average_range, // By Claude on 07/02/2026: was null
null, // average_channels, // average_slice,
clt_parameters.imp.subtract_average, // boolean subtract_average,
clt_parameters.imp.running_average, // int running_average,
null, // fov_tiles, // Rectangle fov_tiles,
1, // mode3d, // int mode3d,
false, // toRGB, // boolean toRGB,
xyz_offset, // double [] stereo_offset, // offset reference camera {x,y,z}
cuas_atr, // double [] stereo_atr, // offset reference orientation (cuas)
sensor_mask, // sensor_mask, // int sensor_mask,
merge_all, // boolean merge_all,
2, // int make_hyper, // By Claude on 07/02/2026: hyperstack, per-timestamp sensor average as first channel
scenes_suffix, // String suffix,
ds_vantage[0], // selected_disparity, // double [] ref_disparity,
scenes, // QuadCLT [] quadCLTs,
getCenter_CLT(), // ref_index, // int ref_index,
ImageDtt.THREADS_MAX, // threadsMax, // int threadsMax,
debugLevel); // int debugLevel);
getCenter_CLT().saveImagePlusInModelDirectory(
imp_targets.getTitle()+"-DBG", // "GPU-SHIFTED-D"+clt_parameters.disparity, // String suffix,
imp_targets); // imp_scenes); // ImagePlus imp)
}
merge_all = true; // clt_parameters.imp.merge_all || !um_mono; // no unsharp mask -> terrain->merge_all
sensor_mask = merge_all? 1 : -1;
ImagePlus imp_targets= OpticalFlow.renderSceneSequence(
clt_parameters, // CLTParameters clt_parameters,
true, // center_CLT.hasCenterClt(), // boolean mode_cuas,
......@@ -149,13 +196,20 @@ public class CuasRanging {
false, // toRGB, // boolean toRGB,
xyz_offset, // double [] stereo_offset, // offset reference camera {x,y,z}
cuas_atr, // double [] stereo_atr, // offset reference orientation (cuas)
1, // sensor_mask, // int sensor_mask,
sensor_mask, // sensor_mask, // int sensor_mask,
merge_all, // boolean merge_all,
0, // int make_hyper, // By Claude on 07/02/2026: flat stack - keep old behavior
scenes_suffix, // String suffix,
ds_vantage[0], // selected_disparity, // double [] ref_disparity,
scenes, // QuadCLT [] quadCLTs,
getCenter_CLT(), // ref_index, // int ref_index,
ImageDtt.THREADS_MAX, // threadsMax, // int threadsMax,
debugLevel); // int debugLevel);
if (dbg_save_fpixels) {
getCenter_CLT().saveImagePlusInModelDirectory(
imp_targets.getTitle()+"-DBG", // "GPU-SHIFTED-D"+clt_parameters.disparity, // String suffix,
imp_targets); // imp_scenes); // ImagePlus imp)
}
return imp_targets;
}
......@@ -2880,8 +2934,66 @@ public class CuasRanging {
getCenter_CLT().saveStringInModelDirectory(tsv.toString(), UAS_DATA_SUFFIX, false);
}
}
// relies on calcMatchingTargetsLengths(.., true,...) called from recalcOmegas() to set [RSLT_GLOBAL]
/** Standalone UAS flight-log extraction for the mode-0 RT path (NO CuasMotion / targets array): project the
* DJI flight log to per-scene px,py,range using center_CLT pose + uasLogReader, accumulate a TSV (same columns
* + UAS_DATA_SUFFIX as addUasData), save via the QuadCLT model dir. Timestamps = imp_targets slice labels
* (skip leading non-digit "average" slices, == CuasDetectRT.ingest). Needs QuadCLT pose, so mode-0 only (not
* the mode=3 file path). By Claude on 06/24/2026 */
public void saveUasFlightLogCsv(UasLogReader uasLogReader, ImagePlus imp_targets) {
if (uasLogReader == null) {
System.out.println("saveUasFlightLogCsv(): no UAS log reader - skipping flight-log CSV");
return;
}
if ((imp_targets == null) || (imp_targets.getStackSize() < 1)) {
System.out.println("saveUasFlightLogCsv(): no imp_targets - skipping flight-log CSV");
return;
}
// camera reference LLA from center_CLT (mirror addUasData)
if ((getCenter_CLT() != null) && getCenter_CLT().hasIns()) {
double [] cameraLla = getCenter_CLT().getLla();
if ((cameraLla != null) && ((cameraLla[0] != 0.0) || (cameraLla[1] != 0.0))) {
uasLogReader.setCameraLLA(cameraLla);
}
}
double [] cam_atr = uasLogReader.getCameraATR();
int tilesX = getCenter_CLT().getTileProcessor().getTilesX();
int tilesY = getCenter_CLT().getTileProcessor().getTilesY();
ij.ImageStack stack = imp_targets.getStack();
int num_slices = stack.getSize();
int first_slice = 1;
for (; (first_slice <= num_slices) && !Character.isDigit(stack.getSliceLabel(first_slice).charAt(0)); first_slice++);
StringBuffer sb = new StringBuffer();
sb.append("seq\tts\tstatus\tpx\tpy\ttile_x\ttile_y\trange\tlat\tlon\talt\tnorth\teast\tdown\tcam_az\tcam_tilt\tcam_roll\n");
int nseq = 0;
for (int slice = first_slice; slice <= num_slices; slice++, nseq++) {
// slice labels can carry a suffix (e.g. "1773135457.547099-0"); extract the bare timestamp the
// same way as QuadCLT.getTimeStamp (regex \d{5,10}\.\d{6}), but keep it a STRING - the double
// round-trip would lose microsecond precision at this magnitude. By Claude on 06/24/2026
String norm = stack.getSliceLabel(slice).replace("_", ".");
java.util.regex.Matcher tm = java.util.regex.Pattern.compile("\\d{5,10}\\.\\d{6}").matcher(norm);
String timestamp = tm.find() ? norm.substring(tm.start(), tm.end()) : norm;
double [] uas = uasLogReader.getUasPxPyDRange(timestamp); // px, py, disparity, range
double [] llaned = uasLogReader.getUasLlaNed(timestamp); // lat, lon, alt, N, E, D
if (uas != null) {
double px = uas[0], py = uas[1], range = uas[3];
int tileX = (int) (px / GPUTileProcessor.DTT_SIZE);
int tileY = (int) (py / GPUTileProcessor.DTT_SIZE);
String status = ((tileX >= 0) && (tileY >= 0) && (tileX < tilesX) && (tileY < tilesY)) ? "IN FoV" : "OUT OF FoV";
sb.append(nseq+"\t"+timestamp+"\t"+status+"\t"+px+"\t"+py+"\t"+tileX+"\t"+tileY+"\t"+range+"\t"+
llaned[0]+"\t"+llaned[1]+"\t"+llaned[2]+"\t"+llaned[3]+"\t"+llaned[4]+"\t"+llaned[5]+"\t"+
cam_atr[0]+"\t"+cam_atr[1]+"\t"+cam_atr[2]+"\n");
} else {
sb.append(nseq+"\t"+timestamp+"\tno entry\t\t\t\t\t\t"+
((llaned != null) ? (llaned[0]+"\t"+llaned[1]+"\t"+llaned[2]) : "\t\t")+
"\t\t\t\t"+cam_atr[0]+"\t"+cam_atr[1]+"\t"+cam_atr[2]+"\n");
}
}
getCenter_CLT().saveStringInModelDirectory(sb.toString(), UAS_DATA_SUFFIX, false);
System.out.println("saveUasFlightLogCsv(): wrote UAS flight log ("+nseq+" scenes) -> "+getCenter_CLT().getImageName()+UAS_DATA_SUFFIX);
}
// relies on calcMatchingTargetsLengths(.., true,...) called from recalcOmegas() to set [RSLT_GLOBAL]
public void saveTargetStats(
final double [][][] targets_single) {
UasLogReader uasLogReader = cuasMotion.getUasLogReader();
......
This diff is collapsed.
package com.elphel.imagej.cuas.rt;
import java.awt.Rectangle;
/**
* DNN inference backend for the CUAS L1+L2 pipeline. Implemented by {@link CuasDnnRemote} (TCP to the
* Python infer_server, e.g. on the DGX) and {@link CuasDnnLocal} (in-process native LibTorch via JNA /
* libtpdnn.so). CuasDetectRT.runDnnRemote() talks to this interface so the two are drop-in swappable;
* the result type {@link CuasDnnRemote.BatchResult} is shared. By Claude on 06/27/2026.
*/
public interface CuasDnnBackend extends AutoCloseable {
/** Upload the conditioned stack frames[T][H*W] (row-major) once; the backend builds the temporal
* pyramid. Returns the per-level frame counts (and sets {@link #getNFrames()}). */
int [] upload(float [][] frames, int H, int W) throws Exception;
/** Temporal depth N of the loaded model (window length), known after {@link #upload}. */
int getNFrames();
/** Infer `count` scenes of a level in one call (newest_s = start + s*stride). rmaxCells>0 enables the
* on-GPU ghostbuster. l2Enable/l2Reset drive Layer-2; noiseScale is the per-level L1-input scale. */
CuasDnnRemote.BatchResult inferBatch(int level, int start, int count, int stride, Rectangle roi,
double rmaxCells, boolean l2Enable, boolean l2Reset, double noiseScale) throws Exception;
@Override
void close();
}
......@@ -38,6 +38,9 @@ public class CuasDnnInfer implements AutoCloseable {
String local = resolveModel(modelSpec);
this.env = OrtEnvironment.getEnvironment();
OrtSession.SessionOptions opts = new OrtSession.SessionOptions();
// Only log errors (not warnings): suppresses the benign dynamic-batch VerifyOutputSizes W-spam
// (reg head's cat pins the graph's declared batch to 1) that was yellow-on-white in Eclipse. By Claude on 06/17/2026
try { opts.setSessionLogLevel(ai.onnxruntime.OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR); } catch (Exception e) {}
// CPU EP by default. GPU later: opts.addCUDA(deviceId) once cuDNN 9 is present.
this.session = env.createSession(local, opts);
this.inputName = session.getInputNames().iterator().next(); // "frames"
......@@ -154,8 +157,12 @@ public class CuasDnnInfer implements AutoCloseable {
}
/** Continuous-velocity (reg) inference over an ROI. For each pixel returns
* {S, Vx, Vy, sigma, dx, dy} from the 6-ch reg head (det, Vx, Vy, logvar, dx, dy):
* S=sigmoid(det), Vx/Vy already bounded by the model, sigma=exp(logvar/2). By Claude on 06/17/2026 */
* {S, Vx, Vy, sigma, dx, dy} from the 6-ch reg head (det/S, Vx, Vy, logvar, dx, dy):
* Vx/Vy already bounded by the model, sigma=exp(logvar/2).
* MF-S models (option a, Andrey 2026-06-18): channel 0 is the RAW matched-filter path-sum S
* (clamp>=0), NOT a det logit -> read raw, no sigmoid (sigmoid would re-saturate the unbounded
* response). S is then the informative vote weight directly (CuasDetectRT.voteScatter). By Claude on 06/18/2026
* DEPRECATED 2026-06-20 — may be revisited for Layer 2 // By Claude */
public double[][] inferROIReg(float[][] frames, int width, int height, Rectangle roi) throws Exception {
final int N = frames.length;
final int P = patch, half = P / 2;
......@@ -187,7 +194,7 @@ public class CuasDnnInfer implements AutoCloseable {
for (int q = 0; q < bs; q++) {
float[][][] op = out[q];
double[] o = out6[base + q];
o[0] = sigmoid(op[0][0][0]); // S
o[0] = Math.max(0.0, op[0][0][0]); // S = raw MF path-sum (clamp>=0); MF-S model, no sigmoid. By Claude on 06/18/2026
o[1] = op[1][0][0]; o[2] = op[2][0][0]; // Vx, Vy (model-bounded)
o[3] = Math.exp(0.5 * op[3][0][0]); // sigma = exp(logvar/2)
o[4] = op[4][0][0]; o[5] = op[5][0][0]; // dx, dy
......
package com.elphel.imagej.cuas.rt;
import java.awt.Rectangle;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.sun.jna.Native;
import com.sun.jna.Pointer;
/**
* In-process native DNN backend: runs the SAME L1+L2 inference as {@link CuasDnnRemote}, but via
* LibTorch through JNA (libtpdnn.so, tile_processor_gpu/jna/tp_dnn.cpp) instead of TCP to a Python
* server. No DGX / no server process. Each model's params (N/P/vr/out_ch, L2 ch_hidden) come from its
* bundled {@code <model>.ts.pt.meta.json} sidecar (single source of truth). By Claude on 06/27/2026.
*
* Model resolution mirrors CuasDnnRemote's bundled-vs-override convention:
* - localDir set -> &lt;localDir&gt;/&lt;model&gt;/model.ts.pt (+ .meta.json) [dev/override]
* - localDir empty -> bundled jar resource /cuas_dnn/&lt;basename&gt;/model.ts.pt extracted to temp
*
* Requires {@code -Djna.library.path=<dir with libtpdnn.so>}; libtpdnn.so finds libtorch via its rpath
* (set by build_dnn.sh) or LD_LIBRARY_PATH.
*/
public class CuasDnnLocal implements CuasDnnBackend {
private static TpDnnJna LIB = null;
private static synchronized TpDnnJna lib() {
if (LIB == null) LIB = Native.load("tpdnn", TpDnnJna.class);
return LIB;
}
private final TpDnnJna lib;
private final Pointer ctx;
private final int N, P, vr, nvel;
private int nFrames = 0;
private int H = 0, W = 0;
private File tmpDir = null; // holds extracted bundled models (deleted on close)
/** @param localDir override dir holding &lt;model&gt;/model.ts.pt, or empty/null to use bundled resources.
* @param l1Model L1 run name (e.g. "runs/weighted9_pm_s").
* @param l2Model L2 run name (e.g. "runs/mexhat_gaps_boost40"), or empty/null for L1-only. */
public CuasDnnLocal(String localDir, String l1Model, String l2Model) throws Exception {
this.lib = lib();
String [] l1 = resolveModel(localDir, l1Model, ".ts.pt"); // {tsPath, metaJson}
this.N = metaInt(l1[1], "N", 9);
this.P = metaInt(l1[1], "P", 24);
this.vr = metaInt(l1[1], "vr", 5);
this.nvel = (2 * vr + 1) * (2 * vr + 1);
String l2ts = "";
int l2ch = 24;
if ((l2Model != null) && !l2Model.isEmpty()) {
String [] l2 = resolveModel(localDir, l2Model, ".l2.ts.pt");
l2ts = l2[0];
l2ch = metaInt(l2[1], "ch_hidden", 24);
}
this.ctx = lib.tpdnn_init(l1[0], l2ts, N, P, vr, l2ch);
if (ctx == null) throw new Exception("tpdnn_init failed (l1=" + l1[0] + ", l2=" + l2ts + ")");
System.out.println("CuasDnnLocal: native LibTorch backend up (L1=" + l1Model
+ ", L2=" + ((l2Model == null || l2Model.isEmpty()) ? "off" : l2Model)
+ "; N=" + N + " P=" + P + " vr=" + vr + " l2_ch=" + l2ch + ")");
}
@Override
public int [] upload(float [][] frames, int H, int W) throws Exception {
this.H = H; this.W = W;
int T = frames.length, hw = H * W;
float [] flat = new float [T * hw]; // [T,H,W] row-major
for (int t = 0; t < T; t++) System.arraycopy(frames[t], 0, flat, t * hw, hw);
lib.tpdnn_upload(ctx, flat, T, H, W);
this.nFrames = N;
int nl = lib.tpdnn_num_levels(ctx);
int [] counts = new int [nl];
for (int i = 0; i < nl; i++) counts[i] = lib.tpdnn_level_frames(ctx, i);
return counts;
}
@Override
public int getNFrames() { return nFrames; }
@Override
public CuasDnnRemote.BatchResult inferBatch(int level, int start, int count, int stride, Rectangle roi,
double rmaxCells, boolean l2Enable, boolean l2Reset, double noiseScale) throws Exception {
int hw = H * W, rn = roi.width * roi.height;
int nchAlloc = l2Enable ? 6 : 5; // native returns 6 with L2 (+age), else 5
float [] o5 = new float [count * nchAlloc * hw];
float [] rfb = new float [count * rn * nvel];
long t0 = System.nanoTime();
int nch = lib.tpdnn_infer(ctx, level, start, count, stride,
roi.x, roi.y, roi.width, roi.height, rmaxCells,
l2Enable ? 1 : 0, l2Reset ? 1 : 0, noiseScale, o5, rfb);
double ms = (System.nanoTime() - t0) / 1.0e6;
CuasDnnRemote.BatchResult r = new CuasDnnRemote.BatchResult();
r.gpuMs = ms; r.H = H; r.W = W; r.count = count; r.nch = nch; r.nvel = nvel;
r.rh = roi.height; r.rw = roi.width;
r.offset5 = new float [count][nch][hw]; // o5 layout [count][nch][hw]
for (int s = 0; s < count; s++)
for (int c = 0; c < nch; c++)
System.arraycopy(o5, (s * nch + c) * hw, r.offset5[s][c], 0, hw);
r.roiField = new float [count][rn][nvel]; // rfb layout [count][rn][nvel]
for (int s = 0; s < count; s++)
for (int p = 0; p < rn; p++)
System.arraycopy(rfb, (s * rn + p) * nvel, r.roiField[s][p], 0, nvel);
return r;
}
@Override
public void close() {
try { if (ctx != null) lib.tpdnn_free(ctx); } catch (Exception e) { /* ignore */ }
if (tmpDir != null) { try { deleteRec(tmpDir); } catch (Exception e) { /* ignore */ } }
}
// ---- model resolution + tiny meta parser ----
/** Returns {tsPath, metaJsonContent} for the model. localDir override, else bundled resource->temp. */
private String [] resolveModel(String localDir, String model, String suffix) throws Exception {
String fileName = "model" + suffix; // model.ts.pt / model.l2.ts.pt
if ((localDir != null) && !localDir.isEmpty()) {
File ts = new File(new File(localDir, model), fileName);
File meta = new File(ts.getPath() + ".meta.json");
if (!ts.exists()) throw new Exception("local model not found: " + ts);
String mj = meta.exists() ? new String(Files.readAllBytes(meta.toPath())) : "";
return new String [] { ts.getPath(), mj };
}
// bundled: /cuas_dnn/<basename(model)>/model.ts.pt (+ .meta.json) -> extract to a temp dir
String base = model.substring(model.lastIndexOf('/') + 1);
if (tmpDir == null) tmpDir = Files.createTempDirectory("cuas_dnn_").toFile();
String res = "/cuas_dnn/" + base + "/" + fileName;
Path tsTmp = extractResource(res, base + suffix);
String mj = "";
try (InputStream is = CuasDnnLocal.class.getResourceAsStream(res + ".meta.json")) {
if (is != null) mj = new String(is.readAllBytes());
}
return new String [] { tsTmp.toString(), mj };
}
private Path extractResource(String res, String tmpName) throws Exception {
try (InputStream is = CuasDnnLocal.class.getResourceAsStream(res)) {
if (is == null) throw new Exception("bundled resource not found: " + res);
Path out = new File(tmpDir, tmpName).toPath();
Files.copy(is, out, StandardCopyOption.REPLACE_EXISTING);
return out;
}
}
private static int metaInt(String json, String key, int dflt) {
if (json == null) return dflt;
Matcher m = Pattern.compile("\"" + key + "\"\\s*:\\s*(-?\\d+)").matcher(json);
return m.find() ? Integer.parseInt(m.group(1)) : dflt;
}
private static void deleteRec(File f) {
File [] kids = f.listFiles();
if (kids != null) for (File k : kids) deleteRec(k);
f.delete();
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
/**
** CuasRtState.java - per-sequence persistent state of the CUAS RT mode (JSON)
**
** Copyright (C) 2026 Elphel, Inc.
**
** -----------------------------------------------------------------------------**
**
** CuasRtState.java is free software: you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation, either version 3 of the License, or
** (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with this program. If not, see <http://www.gnu.org/licenses/>.
** -----------------------------------------------------------------------------**
**
*/
package com.elphel.imagej.cuas.rt;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* Persistent RT-ONLY state of ONE scene sequence: <center_name>-RT-STATE.json in the
* sequence's CENTER model directory (Andrey's ruling 07/05/2026). Scope split:
* - GLOBAL (all sequences) parameters stay with quadCLT_main / corr-xml (saved from
* the GUI, merged with the compare/combine tool);
* - PER-SEQUENCE derived values (vintage-specific calibrations the RT run computes
* and reuses) live HERE, in modern JSON - easy to read, easy to tell apart from
* the legacy parameter files, easy to erase for a forced re-derive (the
* missing-file rule: absent -> recompute+save, present -> reuse).
*
* The class is data-pure: it knows file paths, not QuadCLT - callers (CuasRT, the
* borrow boundary) pass the directory/name. Unknown JSON fields are ignored on load
* (forward compatibility); missing fields stay at their defaults (null = not yet
* derived). Add new RT persistent variables as public fields with a provenance
* string where useful.
*
* By Claude on 07/05/2026, from Andrey's design.
*/
public class CuasRtState {
public static final String SUFFIX = "-RT-STATE.json";
private static final Gson GSON = new GsonBuilder().setPrettyPrinting().serializeNulls().create();
public int version = 1;
public String updated = null; // timestamp of the last save (informational)
// ---- persistent RT variables (per scene sequence) ----
public Double dc_center = null; // adopted borrowed-center DC level (counts); null = not yet derived
public String dc_center_source = null; // provenance, e.g. "auto: sky-half median of the reference render"
/** @return the state file path for this sequence. */
public static String path(String x3d_directory, String center_name) {
return x3d_directory + java.io.File.separator + center_name + SUFFIX;
}
/**
* Load the sequence state, or return a fresh (all-defaults) instance when the file
* is missing or unreadable (unreadable prints a warning - never breaks the run).
*/
public static CuasRtState load(String x3d_directory, String center_name) {
final String path = path(x3d_directory, center_name);
if (!Files.exists(Paths.get(path))) {
return new CuasRtState();
}
try (FileReader reader = new FileReader(path)) {
CuasRtState state = GSON.fromJson(reader, CuasRtState.class);
return (state != null) ? state : new CuasRtState();
} catch (IOException | RuntimeException e) { // malformed JSON -> JsonSyntaxException
System.out.println("CuasRtState.load(): FAILED to read "+path+" ("+e.getMessage()+
") - starting with a fresh state (derived values will be recomputed)");
return new CuasRtState();
}
}
/** Save the sequence state (pretty JSON, stamped). @return true on success. */
public boolean save(String x3d_directory, String center_name) {
final String path = path(x3d_directory, center_name);
this.updated = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date());
try (FileWriter writer = new FileWriter(path)) {
GSON.toJson(this, writer);
System.out.println("CuasRtState.save(): wrote "+path);
return true;
} catch (IOException e) {
System.out.println("CuasRtState.save(): FAILED to write "+path+" ("+e.getMessage()+")");
return false;
}
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -897,6 +897,14 @@ public class ComboMatch {
// for all modes - needed for create_overlaps || process_correlation || render_match || pattern_match
if (GPU_QUAD_AFFINE == null) {
// Fail loud rather than NPE deep in JCuda: the orthomosaic GPU_QUAD_AFFINE (rectilinear) path is
// not on the validated JNA surface yet (wider than CUAS: affine matching). Run orthomosaic on
// JCuda. The CUAS rectilinear path IS ported (GpuQuad.createRectilinear). By Claude on 06/26/2026.
if (GpuQuad.useJnaBackend()) {
throw new UnsupportedOperationException(
"ComboMatch GPU_QUAD_AFFINE (orthomosaic, rectilinear) is not yet ported to the JNA backend "+
"(-Dtp.backend=jna); run orthomosaic with the JCUDA backend.");
}
System.out.println("Setting up GPU");
try {
GPU_QUAD_AFFINE = new GpuQuad(//
......
This diff is collapsed.
This diff is collapsed.
{"kind": "L2", "ch_hidden": 24, "vmax": 1.4, "grid": 32, "ch_in": 3}
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment