CLAUDE: Add DGX remote DNN inference path (full-res shift-and-stitch)

detectTargets() can offload the DNN front-end to the GB10 DGX (curt_dnn_remote): CuasDnnRemote uploads the LoG-conditioned stack once, the DGX builds the pyramid and runs full-res shift-and-stitch per (level,scene), returning the full-frame {dx,dy,s,Vx,Vy} (-OFFSET) + ROI 121-cell softmax*s (-RECT/-HYPER-RECT). Auto-launches the server if down (bundled cuas_dnn/ scripts, or curt_dnn_remote_srcdir local-repo override - mirrors the GPU-kernel default-vs-override). Synthetic targets are mixed into the upload stack so synth works on the remote path. 4 curt_dnn_remote_* dialog params, grouped with the model fields. Local CPU path unchanged (curt_dnn_remote=false) for Layer 2. Validated end-to-end; shift-and-stitch is fp64-exact vs per-pixel.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/main/java/com/elphel/imagej/cuas/rt/CuasDetectRT.java

@@ -662,6 +662,102 @@ public class CuasDetectRT {
 		}
 	}
 
+	/** Remote DGX inference path (curt_dnn_remote): upload the SUBAVG+LoG-conditioned stack once,
+	 *  the DGX builds the pyramid and runs full-res shift-and-stitch per (level, scene), returning the
+	 *  full-frame {dx,dy,s,Vx,Vy} offset (-OFFSET) + the ROI 121-cell softmax*s (-RECT/-HYPER-RECT).
+	 *  Mirrors the local DNN block's level/scene/timing-window indexing; no recurrent feed (the local
+	 *  CPU path, curt_dnn_remote=false, still drives the Layer-2 experiments). Synthetic injection is
+	 *  NOT applied here - it lives on dpixels_pyramid (local path); remote uploads dpixels_log.
+	 *  Server: attic/imagej-elphel-internal/c5p_dnn/infer_server.py. By Claude on 06/20/2026 */
+	private void runDnnRemote(
+			CLTParameters clt_parameters,
+			CuasRTUtils   cuasRTUtils,
+			double [][]   dpixels_log,
+			String [][]   ts_pyramid,
+			int           pyramid_levels,
+			int []        c5_levels,
+			Rectangle     roi,
+			String        title_conv5d) {
+		final int dnn_stride = Math.max(1, clt_parameters.imp.curt_dnn_stride);
+		final int W = getWidth(), H = getHeight();
+		final boolean save_rect  = clt_parameters.imp.curt_save_c5rect;
+		final boolean save_hyper = clt_parameters.imp.curt_save_c5hyper;
+		final boolean synth = clt_parameters.imp.curt_synth_src && (synth_pixels != null) && (synth_pixels.length > 0);
+		final boolean synth_bg = clt_parameters.imp.curt_synth_bg;
+		final int n_synth = synth ? synth_pixels.length : 0;
+		try { // auto-launch the DGX server if not already running (deploy bundled/override scripts, ssh-start, poll) // By Claude on 06/20/2026
+			CuasDnnRemote.ensureServer(clt_parameters.imp.curt_dnn_remote_host, clt_parameters.imp.curt_dnn_remote_model, clt_parameters.imp.curt_dnn_remote_srcdir);
+		} catch (Exception e) { System.out.println("runDnnRemote(): server auto-launch failed: "+e); }
+		try (CuasDnnRemote remote = new CuasDnnRemote(clt_parameters.imp.curt_dnn_remote_host)) {
+			// Build the upload array: LoG-conditioned real (optionally synth_bg_avg-decimated upstream), optionally
+			// with synthetic targets mixed in (tiled synth[t % n_synth]; clean = zero bg first) - matching the local
+			// per-level injection (CuasDetectRT.java synth block) but PRE-pyramid; the DGX then averages it into levels.
+			// So synth now works on the remote path. By Claude on 06/20/2026
+			float [][] logf = new float [dpixels_log.length][];
+			for (int t = 0; t < dpixels_log.length; t++) {
+				double [] s = dpixels_log[t]; float [] f = new float [s.length];
+				if (synth) {
+					float [] sp = synth_pixels[t % n_synth]; int m = Math.min(s.length, sp.length);
+					if (synth_bg) { for (int p = 0; p < s.length; p++) f[p] = (float) s[p]; for (int p = 0; p < m; p++) f[p] += sp[p]; }
+					else          { for (int p = 0; p < m; p++) f[p] = sp[p]; }   // clean: drop real bg, synth only
+				} else {
+					for (int p = 0; p < s.length; p++) f[p] = (float) s[p];
+				}
+				logf[t] = f;
+			}
+			if (synth) System.out.println(now()+" runDnnRemote(): mixed synthetic grid (tiled "+n_synth+" frames"+(synth_bg?", over real bg":", CLEAN - bg zeroed")+") into the upload stack");
+			long tup = System.currentTimeMillis();
+			int [] levCounts = remote.upload(logf, H, W);
+			int N_dnn = remote.getNFrames();
+			System.out.println(now()+" runDnnRemote(): UPLOAD "+dpixels_log.length+" frames -> levels "
+					+ java.util.Arrays.toString(levCounts) + " N="+N_dnn
+					+ "  ("+(System.currentTimeMillis()-tup)+" ms, "+(dpixels_log.length*(long)H*W*4/1000000)+" MB)");
+			int nlevels = Math.min(pyramid_levels, levCounts.length);
+			for (int nlev = 0; nlev < nlevels; nlev++) {
+				if (!c5LevelSelected(c5_levels, nlev) || (ts_pyramid[nlev] == null)) continue;
+				int levLen = levCounts[nlev];
+				int num_all = (levLen >= N_dnn) ? ((levLen - N_dnn)/dnn_stride + 1) : 0;
+				if (num_all == 0) continue;
+				// timing ROI (curt_time_from/to) on the level timestamps - infer only the in-window scenes
+				String [] cand_ts = new String [num_all];
+				for (int k = 0; k < num_all; k++) cand_ts[k] = ts_pyramid[nlev][k*dnn_stride + N_dnn - 1];
+				int [] tw = timeWindow(cand_ts); int w0 = tw[0], num = tw[1]-tw[0];
+				if (num <= 0) continue;
+				int rnp = roi.width * roi.height;
+				double [][][][] dnn_roi = new double [num][][][];     // [scene][roi_pix][1][nvel] -> -RECT/-HYPER-RECT
+				double [][][]   off5    = new double [5][num][H*W];    // {dx,dy,s,Vx,Vy} full-frame -> -OFFSET
+				String [] ts_dnn = new String [num];
+				System.out.println(now()+" runDnnRemote(): LEV"+nlev+" "+num+" of "+num_all+" scenes, stride "+dnn_stride+", ROI "+roi.width+"x"+roi.height);
+				for (int j = 0; j < num; j++) {
+					int newest = (w0 + j)*dnn_stride + N_dnn - 1;     // newest frame index in this level (same as local)
+					long t0 = System.currentTimeMillis();
+					CuasDnnRemote.Result r = remote.infer(nlev, newest, roi);
+					long t1 = System.currentTimeMillis();
+					double [][][] fld = new double [rnp][1][r.nvel];
+					for (int p = 0; p < rnp; p++) { float [] rv = r.roiField[p]; double [] dv = fld[p][0]; for (int v = 0; v < r.nvel; v++) dv[v] = rv[v]; }
+					dnn_roi[j] = fld;
+					for (int c = 0; c < 5; c++) { float [] sc = r.offset5[c]; double [] dc = off5[c][j]; for (int p = 0; p < H*W; p++) dc[p] = sc[p]; }
+					ts_dnn[j] = ts_pyramid[nlev][newest] + " f"+newest;
+					System.out.println(now()+"   DNN-remote scene "+(j+1)+"/"+num+" (f"+newest+"): gpu_exec="+d2s(r.gpuMs)+"ms roundtrip="+(t1-t0)+"ms");
+				}
+				String roiTag = "-ROI"+roi.x+"_"+roi.y+"_"+roi.width+"_"+roi.height;
+				String title = title_conv5d+"-DNN"+((nlev>0)?("-LEV"+nlev):"")+roiTag;
+				int [] win_dnn = timeWindow(ts_dnn);
+				double [][][][] dnn_w = win4(dnn_roi, win_dnn); String [] ts_w = winS(ts_dnn, win_dnn);
+				if (save_rect)  QuadCLTCPU.saveImagePlusInDirectory(tagCuasImp(cuasRTUtils.showConvKernel5d(         dnn_w, roi, ts_w, title+"-RECT"),       clt_parameters.imp), getModelDirectory());
+				if (save_hyper) QuadCLTCPU.saveImagePlusInDirectory(tagCuasImp(cuasRTUtils.showConvKernel5dHyperRect(dnn_w, roi, ts_w, title+"-HYPER-RECT"), clt_parameters.imp), getModelDirectory());
+				int nsc = win_dnn[1]-win_dnn[0];
+				double [][][] off5_w = new double [5][nsc][];          // window the full-frame offset for -OFFSET
+				for (int c = 0; c < 5; c++) for (int k = 0; k < nsc; k++) off5_w[c][k] = off5[c][win_dnn[0]+k];
+				QuadCLTCPU.saveImagePlusInDirectory(tagCuasImp(ShowDoubleFloatArrays.showArraysHyperstack(
+						off5_w, W, title+"-OFFSET", ts_w, new String[]{"dx","dy","s","Vx","Vy"}, false), clt_parameters.imp), getModelDirectory());
+				System.out.println(now()+" runDnnRemote(): LEV"+nlev+" saved -RECT/-HYPER-RECT (ROI) + -OFFSET (full "+W+"x"+H+", {dx,dy,s,Vx,Vy})");
+			}
+		} catch (Exception e) {
+			System.out.println("runDnnRemote() failed: "+e); e.printStackTrace();
+		}
+	}
+
 	public CuasMotion detectTargets(
 			CLTParameters     clt_parameters,
 			boolean           batch_mode,
@@ -935,7 +1031,7 @@ public class CuasDetectRT {
    		// same shape as the C5P output, saved as -DNN-RECT / -DNN-HYPER-RECT. N (temporal
    		// depth, 8 or 9) is read from the loaded ONNX. Reuses curt_c5_levels for level gating. Recurrent feed is the next step
    		// (the field is [0,1]-scaled, unlike the C5P matched-filter response - needs rescale).
-   		if (!clt_parameters.imp.curt_dnn_model.isEmpty() && (curt_save_select != null)) {     // By Claude on 06/13/2026
+   		if (!clt_parameters.imp.curt_dnn_model.isEmpty() && (curt_save_select != null) && !clt_parameters.imp.curt_dnn_remote) { // local CPU path (remote = DGX block below) // By Claude on 06/13/2026, remote guard 06/20/2026
    			final int vr_dnn = clt_parameters.imp.curt_vel_radius;                            // By Claude on 06/13/2026
    			final int dnn_stride = Math.max(1, clt_parameters.imp.curt_dnn_stride);           // 1 = every slice (testing), 4 = production 50% overlap // By Claude on 06/14/2026
    			try {                                                                             // By Claude on 06/13/2026
@@ -1101,6 +1197,10 @@ public class CuasDetectRT {
    				System.out.println("DNN inference failed: "+e); e.printStackTrace();          // By Claude on 06/13/2026
    			}                                                                                 // By Claude on 06/13/2026
    		}                                                                                     // By Claude on 06/13/2026
+   		// Remote DGX inference path (curt_dnn_remote): upload conditioned stack, DGX builds pyramid + infers. // By Claude on 06/20/2026
+   		if ((curt_save_select != null) && clt_parameters.imp.curt_dnn_remote) { // model lives on the DGX (run dir) - no local curt_dnn_model needed // By Claude on 06/20/2026
+   			runDnnRemote(clt_parameters, cuasRTUtils, dpixels_log, ts_pyramid, pyramid_levels, c5_levels, curt_save_select, title_conv5d); // By Claude on 06/20/2026
+   		}
    		System.out.println(now()+" detectTargets(): done"); // By Claude on 06/11/2026
    		return null;
    		/*

src/main/java/com/elphel/imagej/cuas/rt/CuasDnnRemote.java

+package com.elphel.imagej.cuas.rt;
+
+import java.awt.Rectangle;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+
+/**
+ * TCP client for the DGX remote DNN inference server (attic/imagej-elphel-internal/c5p_dnn/infer_server.py).
+ * By Claude on 06/20/2026.
+ *
+ * Stateful, big-endian protocol (Java DataInput/OutputStream is big-endian; the Python side uses
+ * struct '>' / numpy '>f4', so they match without byte-swapping):
+ *   UPLOAD  : cmd=1, int T,H,W, then T*H*W float32 (the SUBAVG+LoG-conditioned stack)
+ *             -> reply: int n_levels, n_levels x int frames_per_level, int N, double build_ms
+ *             The DGX builds the temporal pyramid (0.5*(now+prev), == temporalAverageLReLU).
+ *   INFER   : cmd=2, int level, newest, roi_x, roi_y, roi_w, roi_h
+ *             -> reply: double gpu_ms, int H,W, 5*H*W float32 (offset5 dx,dy,s,Vx,Vy plane-major),
+ *                       int rh,rw,nvel, rh*rw*nvel float32 (ROI softmax*s, pixel-major)
+ *   READBACK: cmd=3, int level, frame -> reply: int H,W, H*W float32 (one conditioned pyramid frame)
+ *   BYE     : cmd=0
+ */
+public class CuasDnnRemote implements AutoCloseable {
+	private static final int CMD_BYE = 0, CMD_UPLOAD = 1, CMD_INFER = 2, CMD_READBACK = 3;
+	private final Socket sock;
+	private final DataInputStream  in;
+	private final DataOutputStream out;
+	private int nFrames = 0;   // temporal depth N, learned from the UPLOAD reply
+
+	public CuasDnnRemote(String hostPort) throws Exception {
+		String [] hp = hostPort.split(":");
+		int port = (hp.length > 1) ? Integer.parseInt(hp[1].trim()) : 5577;
+		sock = new Socket(hp[0].trim(), port);
+		sock.setTcpNoDelay(true);
+		in  = new DataInputStream (new BufferedInputStream (sock.getInputStream(),  1 << 20));
+		out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream(), 1 << 20));
+	}
+
+	public int getNFrames() { return nFrames; }
+
+	/** Upload the conditioned stack frames[T][H*W] (row-major) once; the DGX builds the pyramid.
+	 *  Returns the per-level frame counts (and sets getNFrames()). */
+	public int [] upload(float [][] frames, int H, int W) throws Exception {
+		int T = frames.length;
+		out.writeInt(CMD_UPLOAD); out.writeInt(T); out.writeInt(H); out.writeInt(W);
+		byte [] buf = new byte [H * W * 4];                 // one frame, reused
+		ByteBuffer bb = ByteBuffer.wrap(buf);               // big-endian by default
+		for (int t = 0; t < T; t++) {
+			bb.clear();
+			float [] fr = frames[t];
+			for (int p = 0; p < fr.length; p++) bb.putFloat(fr[p]);
+			out.write(buf);
+		}
+		out.flush();
+		int nl = in.readInt();
+		int [] counts = new int [nl];
+		for (int i = 0; i < nl; i++) counts[i] = in.readInt();
+		nFrames = in.readInt();
+		in.readDouble();                                    // build_ms (logged server-side)
+		return counts;
+	}
+
+	/** One INFER result: full-frame offset5 {dx,dy,s,Vx,Vy}[5][H*W] (plane-major) +
+	 *  ROI field [rh*rw][nvel] (softmax*s, pixel-major) + the GPU compute ms. */
+	public static class Result {
+		public double gpuMs;
+		public int H, W, rh, rw, nvel;
+		public float [][] offset5;    // [5][H*W]
+		public float [][] roiField;   // [rh*rw][nvel]
+	}
+
+	public Result infer(int level, int newest, Rectangle roi) throws Exception {
+		out.writeInt(CMD_INFER); out.writeInt(level); out.writeInt(newest);
+		out.writeInt(roi.x); out.writeInt(roi.y); out.writeInt(roi.width); out.writeInt(roi.height);
+		out.flush();
+		Result r = new Result();
+		r.gpuMs = in.readDouble(); r.H = in.readInt(); r.W = in.readInt();
+		int hw = r.H * r.W;
+		byte [] ob = new byte [5 * hw * 4]; in.readFully(ob);
+		ByteBuffer obb = ByteBuffer.wrap(ob);
+		r.offset5 = new float [5][hw];
+		for (int c = 0; c < 5; c++) for (int p = 0; p < hw; p++) r.offset5[c][p] = obb.getFloat();
+		r.rh = in.readInt(); r.rw = in.readInt(); r.nvel = in.readInt();
+		int rn = r.rh * r.rw;
+		byte [] rb = new byte [rn * r.nvel * 4]; in.readFully(rb);
+		ByteBuffer rbb = ByteBuffer.wrap(rb);
+		r.roiField = new float [rn][r.nvel];
+		for (int p = 0; p < rn; p++) for (int v = 0; v < r.nvel; v++) r.roiField[p][v] = rbb.getFloat();
+		return r;
+	}
+
+	/** Debug: read back one conditioned pyramid frame [H*W] (sets hw[0]=H, hw[1]=W if provided). */
+	public float [] readback(int level, int frame, int [] hw) throws Exception {
+		out.writeInt(CMD_READBACK); out.writeInt(level); out.writeInt(frame); out.flush();
+		int H = in.readInt(), W = in.readInt();
+		if ((hw != null) && (hw.length >= 2)) { hw[0] = H; hw[1] = W; }
+		byte [] b = new byte [H * W * 4]; in.readFully(b);
+		ByteBuffer bb = ByteBuffer.wrap(b);
+		float [] f = new float [H * W];
+		for (int i = 0; i < f.length; i++) f[i] = bb.getFloat();
+		return f;
+	}
+
+	@Override
+	public void close() {
+		try { out.writeInt(CMD_BYE); out.flush(); } catch (Exception e) { /* ignore */ }
+		try { sock.close(); } catch (Exception e) { /* ignore */ }
+	}
+
+	// ---- Auto-launch (no manual interaction): ensure the DGX server is running, else deploy + start it ----
+	// Mirrors the GPU-kernel default-vs-override: server scripts come from the bundled jar resource
+	// (/cuas_dnn/...) unless srcdir is set, then from that local dir. By Claude on 06/20/2026
+
+	private static boolean canConnect(String host, int port, int timeoutMs) {
+		try (Socket s = new Socket()) { s.connect(new InetSocketAddress(host, port), timeoutMs); return true; }
+		catch (Exception e) { return false; }
+	}
+
+	/** Read a server script: bundled jar resource (/cuas_dnn/<name>) by default, or <srcdir>/<name> if set. */
+	private static byte [] readServerScript(String name, String srcdir) throws Exception {
+		if ((srcdir != null) && !srcdir.isEmpty()) {
+			return Files.readAllBytes(new File(srcdir, name).toPath());           // local-repo override (config)
+		}
+		try (InputStream is = CuasDnnRemote.class.getResourceAsStream("/cuas_dnn/" + name)) {
+			if (is == null) throw new Exception("bundled resource /cuas_dnn/" + name + " not found");
+			return is.readAllBytes();                                             // working version in jar
+		}
+	}
+
+	private static void deployScript(String name, String srcdir, String sshTarget, String dest) throws Exception {
+		byte [] content = readServerScript(name, srcdir);
+		ProcessBuilder pb = new ProcessBuilder("ssh", sshTarget, "cat > " + dest);
+		pb.redirectErrorStream(true);
+		Process p = pb.start();
+		try (OutputStream os = p.getOutputStream()) { os.write(content); }
+		if (p.waitFor() != 0) throw new Exception("deploy " + name + " to " + sshTarget + ":" + dest + " failed");
+	}
+
+	/** Ensure the DGX server is reachable at hostPort; if not, deploy the (bundled/override) scripts and
+	 *  ssh-launch run_infer_server.sh with RUN=model, then poll until it accepts connections. No manual step. */
+	public static void ensureServer(String hostPort, String model, String srcdir) throws Exception {
+		String [] hp = hostPort.split(":");
+		String host = hp[0].trim();
+		int port = (hp.length > 1) ? Integer.parseInt(hp[1].trim()) : 5577;
+		if (canConnect(host, port, 1500)) return;                                 // already up
+		String sshTarget = "elphel@" + host;                                      // DGX login user
+		String code = "/home/elphel/c5p_dnn";                                     // DGX dir (model.py + runs/ live here)
+		System.out.println("CuasDnnRemote.ensureServer(): no server at " + host + ":" + port
+				+ " - deploying scripts + launching on " + sshTarget + " (model=" + model + ")");
+		deployScript("infer_server.py",     srcdir, sshTarget, code + "/infer_server.py");
+		deployScript("run_infer_server.sh",  srcdir, sshTarget, code + "/run_infer_server.sh");
+		ProcessBuilder pb = new ProcessBuilder("ssh", sshTarget,
+				"cd " + code + " && chmod +x run_infer_server.sh && RUN=" + model + " PORT=" + port + " ./run_infer_server.sh start");
+		pb.inheritIO();
+		pb.start().waitFor();
+		long deadline = System.currentTimeMillis() + 90000;                       // model load + warm-up can take a bit
+		while (System.currentTimeMillis() < deadline) {
+			if (canConnect(host, port, 1500)) { System.out.println("CuasDnnRemote.ensureServer(): server up at " + host + ":" + port); return; }
+			Thread.sleep(2000);
+		}
+		throw new Exception("CuasDnnRemote.ensureServer(): server did not come up at " + host + ":" + port);
+	}
+}

src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java

@@ -1164,6 +1164,10 @@ min_str_neib_fpn	0.35
 	public double   curt_dnn_vmax =        1.4;   // "ghostbuster": trained DNN velocity limit (px/frame); zero velocity-grid cells beyond this radius (cell-R > vmax*vel_decimate) and discard pixels whose peak lands there (s=0) - the untrained corner cells emit spurious sidelobes that would confuse the recurrent. Set to match the loaded model's training vmax_px (PM models=1.4, base=1.0); <=0 disables // By Claude on 06/15/2026
 	// dnn_t8frac (T-8 ghost filter) removed 2026-06-20; predecessor code at git tag cuas-layer1. // By Claude on 06/20/2026
 	public int      curt_dnn_patch =       24;    // DNN receptive-field / training patch size (px): inferROI extracts this PxP patch per ROI pixel (half=P/2 = the output pixel). MUST match the loaded model (24 = base/PM models, 32 = larger-attention model) // By Claude on 06/16/2026
+	public boolean  curt_dnn_remote =      false; // run the DNN front-end on the remote DGX server (CuasDnnRemote) instead of local CPU ORT: upload the conditioned stack once, DGX builds the pyramid + full-res shift-and-stitch, returns full-frame {dx,dy,s,Vx,Vy} (-OFFSET) + ROI 121-cell softmax*s (-RECT/-HYPER-RECT). Local CPU path + recurrent feed untouched when off. // By Claude on 06/20/2026
+	public String   curt_dnn_remote_host = "192.168.0.62:5577"; // DGX inference server host:port for curt_dnn_remote (see attic/imagej-elphel-internal/c5p_dnn/infer_server.py) // By Claude on 06/20/2026
+	public String   curt_dnn_remote_model = "runs/weighted9_pm_s"; // DGX-side run dir (with model.pt) the auto-launched server loads (passed as RUN=) // By Claude on 06/20/2026
+	public String   curt_dnn_remote_srcdir = ""; // server-scripts override dir: empty = bundled jar resource (cuas_dnn/), set = local dir - same default-vs-override scheme as the GPU kernels (cuda_project_directory) // By Claude on 06/20/2026
 	public boolean  curt_dnn_recur_splat = false; // when feeding the DNN field to the recurrent layer: false = feed per-pixel field as-is; true = splat each pixel's velocity vector to its fractional offset (px+dx,py+dy) so neighbours reinforce in one sub-pixel bin // By Claude on 06/14/2026
 	public double   curt_dnn_recur_scale = 10.0;  // multiply the DNN field (softmax*s, peaks ~0.1) by this before the recurrent feed, to reach the recurrent's tuned scale (rs_min=1.0); ~10 -> peak ~1.0. Alternative to lowering curt_recur_rs_min // By Claude on 06/14/2026
 	public boolean  curt_synth_src =       true;  // default set for the synthetic B-measurement experiment (set false for real-data runs); reads *-CUAS-SYNTHETIC-CUAS.tiff, output titles get -SYNTH // By Claude on 06/12/2026
@@ -3466,6 +3470,14 @@ min_str_neib_fpn	0.35
 				"Trained DNN front-end model location. Empty = disabled (use matched-filter/posterior path). Local path, scp user@host:/path (fetched to cache), or http(s) URL. Overrides any bundled resource - same default-vs-override scheme as the GPU kernel sources."); // By Claude on 06/13/2026
 		gd.addStringField ("C5P Stage-2 vote-refine (ONNX)",         this.curt_stage2_model, 60, // By Claude on 06/18/2026
 				"v2 Stage-2 learned Hough-vote refine model. Empty = Stage-1 reg field only. When set (with a reg Stage-1 model) the reg branch adds the vote heatmap (-VOTE) and refined detection (-REFINED) views.");
+		gd.addCheckbox    ("DNN remote (run on DGX)",                this.curt_dnn_remote, // grouped with the model fields // By Claude on 06/20/2026
+				"Run the DNN on the remote DGX inference server instead of local CPU: upload the conditioned stack once, the DGX builds the pyramid + full-res shift-and-stitch, returns full-frame {dx,dy,s,Vx,Vy} (-OFFSET) and ROI 121-cell softmax*s (-RECT/-HYPER-RECT). Local CPU path (+ recurrent feed) used when off."); // By Claude on 06/20/2026
+		gd.addStringField ("DNN remote host:port",                   this.curt_dnn_remote_host, 24, // By Claude on 06/20/2026
+				"DGX inference server address for 'DNN remote' (default 192.168.0.62:5577)."); // By Claude on 06/20/2026
+		gd.addStringField ("DNN remote model (DGX run dir)",         this.curt_dnn_remote_model, 24, // By Claude on 06/20/2026
+				"DGX-side run directory (containing model.pt) the auto-launched server loads, e.g. runs/weighted9_pm_s.");
+		gd.addStringField ("DNN remote server src (empty=bundled)",  this.curt_dnn_remote_srcdir, 40, // By Claude on 06/20/2026
+				"Override dir for the DGX server scripts (infer_server.py / run_infer_server.sh): empty = bundled jar resource (cuas_dnn/); set = local dir. Same default-vs-override (bundled resource vs local repo) scheme as the GPU kernels - bundled is the working version, refresh it after server-script dev.");
 		gd.addNumericField("DNN confidence threshold",               this.curt_dnn_thresh, 6,8,"", // By Claude on 06/13/2026
 				"Zero the DNN (Vx,Vy,s) field where detection confidence s < this (0 = show all). Display/FP-suppression only - real-clutter false positives need a retrain with real-background negatives, not a threshold."); // By Claude on 06/13/2026
 		gd.addCheckbox ("DNN recurrent feed: offset-splat",          this.curt_dnn_recur_splat, // By Claude on 06/14/2026
@@ -5010,6 +5022,10 @@ min_str_neib_fpn	0.35
 		this.curt_c5_levels =           StringToInts(gd.getNextString()); // By Claude on 06/13/2026
 		this.curt_dnn_model =           gd.getNextString().trim(); // By Claude on 06/13/2026
 		this.curt_stage2_model =        gd.getNextString().trim(); // By Claude on 06/18/2026
+		this.curt_dnn_remote =          gd.getNextBoolean();     // grouped with the model fields // By Claude on 06/20/2026
+		this.curt_dnn_remote_host =     gd.getNextString().trim(); // By Claude on 06/20/2026
+		this.curt_dnn_remote_model =    gd.getNextString().trim(); // By Claude on 06/20/2026
+		this.curt_dnn_remote_srcdir =   gd.getNextString().trim(); // By Claude on 06/20/2026
 		this.curt_dnn_thresh =          gd.getNextNumber();      // By Claude on 06/13/2026
 		this.curt_dnn_recur_splat =     gd.getNextBoolean();     // By Claude on 06/14/2026
 		this.curt_dnn_recur_scale =     gd.getNextNumber();      // By Claude on 06/14/2026
@@ -6373,6 +6389,10 @@ min_str_neib_fpn	0.35
 		properties.setProperty(prefix+"curt_dnn_vmax",        this.curt_dnn_vmax+"");       // double // By Claude on 06/15/2026
 		// dnn_t8frac setProperty removed 2026-06-20; predecessor code at git tag cuas-layer1. // By Claude on 06/20/2026
 		properties.setProperty(prefix+"curt_dnn_patch",       this.curt_dnn_patch+"");      // int // By Claude on 06/16/2026
+		properties.setProperty(prefix+"curt_dnn_remote",      this.curt_dnn_remote+"");     // boolean // By Claude on 06/20/2026
+		properties.setProperty(prefix+"curt_dnn_remote_host", this.curt_dnn_remote_host);   // String // By Claude on 06/20/2026
+		properties.setProperty(prefix+"curt_dnn_remote_model", this.curt_dnn_remote_model); // String // By Claude on 06/20/2026
+		properties.setProperty(prefix+"curt_dnn_remote_srcdir", this.curt_dnn_remote_srcdir); // String // By Claude on 06/20/2026
 		properties.setProperty(prefix+"curt_synth_src",       this.curt_synth_src+"");      // boolean // By Claude on 06/11/2026
 		properties.setProperty(prefix+"curt_synth_scale",     this.curt_synth_scale+"");    // double  // By Claude on 06/12/2026
 		properties.setProperty(prefix+"curt_synth_bg_avg",    this.curt_synth_bg_avg+"");   // int     // By Claude on 06/20/2026
@@ -6764,6 +6784,10 @@ min_str_neib_fpn	0.35
 		if (properties.getProperty(prefix+"curt_dnn_vmax")!=null)        this.curt_dnn_vmax=Double.parseDouble(properties.getProperty(prefix+"curt_dnn_vmax")); // By Claude on 06/15/2026
 		// dnn_t8frac getProperty removed 2026-06-20; predecessor code at git tag cuas-layer1. // By Claude on 06/20/2026
 		if (properties.getProperty(prefix+"curt_dnn_patch")!=null)       this.curt_dnn_patch=Integer.parseInt(properties.getProperty(prefix+"curt_dnn_patch")); // By Claude on 06/16/2026
+		if (properties.getProperty(prefix+"curt_dnn_remote")!=null)      this.curt_dnn_remote=Boolean.parseBoolean(properties.getProperty(prefix+"curt_dnn_remote")); // By Claude on 06/20/2026
+		if (properties.getProperty(prefix+"curt_dnn_remote_host")!=null) this.curt_dnn_remote_host=(String) properties.getProperty(prefix+"curt_dnn_remote_host"); // By Claude on 06/20/2026
+		if (properties.getProperty(prefix+"curt_dnn_remote_model")!=null)  this.curt_dnn_remote_model=(String) properties.getProperty(prefix+"curt_dnn_remote_model"); // By Claude on 06/20/2026
+		if (properties.getProperty(prefix+"curt_dnn_remote_srcdir")!=null) this.curt_dnn_remote_srcdir=(String) properties.getProperty(prefix+"curt_dnn_remote_srcdir"); // By Claude on 06/20/2026
 
 		if (properties.getProperty(prefix+"curt_synth_src")!=null)       this.curt_synth_src=Boolean.parseBoolean(properties.getProperty(prefix+"curt_synth_src"));         // By Claude on 06/11/2026
 		if (properties.getProperty(prefix+"curt_synth_scale")!=null)     this.curt_synth_scale=Double.parseDouble(properties.getProperty(prefix+"curt_synth_scale"));       // By Claude on 06/12/2026
@@ -9037,6 +9061,10 @@ min_str_neib_fpn	0.35
 		imp.curt_dnn_vmax =         this.curt_dnn_vmax;     // By Claude on 06/15/2026
 		// dnn_t8frac imp.X copy removed 2026-06-20; predecessor code at git tag cuas-layer1. // By Claude on 06/20/2026
 		imp.curt_dnn_patch =        this.curt_dnn_patch;    // By Claude on 06/16/2026
+		imp.curt_dnn_remote =       this.curt_dnn_remote;    // By Claude on 06/20/2026
+		imp.curt_dnn_remote_host =  this.curt_dnn_remote_host; // By Claude on 06/20/2026
+		imp.curt_dnn_remote_model = this.curt_dnn_remote_model; // By Claude on 06/20/2026
+		imp.curt_dnn_remote_srcdir = this.curt_dnn_remote_srcdir; // By Claude on 06/20/2026
 		imp.curt_synth_src =        this.curt_synth_src;     // By Claude on 06/11/2026
 		imp.curt_synth_scale =      this.curt_synth_scale;   // By Claude on 06/12/2026
 		imp.curt_synth_bg_avg =     this.curt_synth_bg_avg;  // By Claude on 06/20/2026

src/main/resources/cuas_dnn/infer_server.py

+#!/usr/bin/env python3
+"""DGX remote-inference server for the CUAS RawFCN (stateful). By Claude on 06/20/2026.
+
+Java uploads the SUBAVG+LoG-conditioned stack ONCE; the DGX builds the temporal pyramid
+(the 0.5*(now+prev) averaging, exactly as Java's temporalAverageLReLU), then serves
+full-resolution dense inference per (level, scene): shift-and-stitch -> decode on GPU ->
+return only the FULL-FRAME offset {dx,dy,s,Vx,Vy} + the ROI-only 121-cell softmax*s field.
+A debug READBACK returns one conditioned pyramid frame for inspection. Runs inside
+nvcr.io/nvidia/pytorch:25.10-py3.
+
+Protocol (big-endian, matches Java DataInput/OutputStream). Each request: int32 cmd, then:
+  UPLOAD  (1): int32 T,H,W ; T*H*W float32 (conditioned stack)
+              -> reply: int32 n_levels ; n_levels x int32 frames_per_level ; float64 build_ms
+  INFER   (2): int32 level, newest, roi_x, roi_y, roi_w, roi_h
+              -> reply: float64 gpu_ms ; int32 H,W ; 5*H*W float32 (offset5: dx,dy,s,Vx,Vy, plane-major)
+                        int32 rh,rw,nvel ; rh*rw*nvel float32 (ROI softmax*s, pixel-major)
+  READBACK(3): int32 level, frame  -> reply: int32 H,W ; H*W float32 (one conditioned frame)
+  BYE     (0): close
+"""
+import argparse, os, socket, struct, time
+from datetime import datetime
+import numpy as np
+import torch
+import torch.nn.functional as F
+from model import RawFCN
+
+CMD_BYE, CMD_UPLOAD, CMD_INFER, CMD_READBACK = 0, 1, 2, 3
+
+
+def load_model(run_dir, device):
+    ck = torch.load(os.path.join(run_dir, "model.pt"), map_location="cpu", weights_only=False)
+    a = ck.get("args", {}) or {}
+    kw = dict(n_frames=a.get("nframes", 8), vel_radius=a.get("vel_radius", 5),
+              patch=a.get("patch", 24), velocity_mode=a.get("velocity_mode", "grid"),
+              vmax=a.get("vmax", 1.4))
+    if a.get("ch") is not None:
+        kw["ch"] = tuple(a["ch"])
+    m = RawFCN(**kw)
+    m.load_state_dict(ck["model"])
+    m.eval().to(device)
+    print(f"loaded {run_dir}/model.pt: N={kw['n_frames']} patch={kw['patch']} vr={kw['vel_radius']} "
+          f"mode={kw['velocity_mode']} out_ch={m.out_ch}", flush=True)
+    return m, kw["n_frames"], kw["patch"], kw["vel_radius"]
+
+
+def recvall(conn, n):
+    buf = bytearray()
+    while len(buf) < n:
+        chunk = conn.recv(n - len(buf))
+        if not chunk:
+            raise ConnectionError("short read")
+        buf += chunk
+    return bytes(buf)
+
+
+def build_pyramid(log, n_levels_max=8):
+    # Replicates Java's pyramid (CuasDetectRT.java:824-883, temporalAverageLReLU linear).
+    # log: [T,H,W] LoG-conditioned frames.
+    #   level0[i]   = 0.5*(log[i+1] + log[i])                 i=0..T-2          (slice-as-shifted-view add)
+    #   level L+1[i]= 0.5*(levelL[2i+2] + levelL[2i])         i=0..len(L)//2-2
+    levels = [0.5 * (log[1:] + log[:-1])]                     # [T-1, H, W]
+    while len(levels) < n_levels_max:
+        prev = levels[-1]                                     # [Tl, H, W]
+        nl = len(prev) // 2 - 1
+        if nl < 1:
+            break
+        idx = torch.arange(nl, device=log.device)            # [nl]
+        levels.append(0.5 * (prev[2 * idx + 2] + prev[2 * idx]))  # [nl, H, W]
+    return levels
+
+
+@torch.no_grad()
+def shift_stitch(m, x, P, S=4):
+    # FULL-RES field via shift-and-stitch (validated == per-pixel in fp64, dense_check.py).
+    # x: [N,H,W]. Pad by half so dense cell (oi,oj) aligns to input pixel (S*oi,S*oj); the
+    # S*S shifts interleave into [C,H,W].
+    N, H, W = x.shape
+    half = P // 2
+    xp = F.pad(x, (half, half, half, half))                   # [N, H+P, W+P]
+    full = torch.zeros(m.out_ch, H, W, device=x.device, dtype=x.dtype)
+    for sy in range(S):
+        for sx in range(S):
+            y = m(xp[:, sy:, sx:][None])[0]                   # [C, oH, oW]
+            oh = min(y.shape[1], (H - sy + S - 1) // S)
+            ow = min(y.shape[2], (W - sx + S - 1) // S)
+            full[:, sy::S, sx::S] = y[:, :oh, :ow]
+    return full                                               # [C, H, W]
+
+
+@torch.no_grad()
+def decode(field, vr, roi):
+    # field: [C,H,W] full-res. C = 1(det) + nvel + 2(dx,dy). Decode on GPU.
+    vdim = 2 * vr + 1
+    nvel = vdim * vdim
+    C, H, W = field.shape
+    s = field[0].sigmoid()                                    # [H,W]   detection confidence
+    p = field[1:1 + nvel].softmax(0)                          # [nvel,H,W] velocity softmax
+    # cell k -> (vx,vy) = (k%vdim - vr, k//vdim - vr); centroid = sum_k p_k * coord_k
+    k = torch.arange(nvel, device=field.device)
+    cellx = (k % vdim - vr).to(field.dtype)                   # [nvel]
+    celly = (k // vdim - vr).to(field.dtype)
+    vx = (p * cellx[:, None, None]).sum(0)                    # [H,W]  velocity centroid (cells)
+    vy = (p * celly[:, None, None]).sum(0)
+    dx, dy = field[1 + nvel], field[1 + nvel + 1]            # [H,W]
+    offset5 = torch.stack([dx, dy, s, vx, vy])               # [5,H,W]  -> -OFFSET (full frame)
+    x0, y0, rw, rh = roi
+    roi_field = (p * s[None])[:, y0:y0 + rh, x0:x0 + rw]     # [nvel,rh,rw]  softmax*s over ROI
+    return offset5, roi_field.permute(1, 2, 0).contiguous(), nvel  # offset5 [5,H,W], roi [rh,rw,nvel]
+
+
+def serve(run_dir, host, port):
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    torch.backends.cudnn.benchmark = True
+    m, N, P, vr = load_model(run_dir, device)
+    print(f"device={device} gpu={torch.cuda.get_device_name(0) if device=='cuda' else 'cpu'} "
+          f"patch={P} N={N} vr={vr}", flush=True)
+    _ = shift_stitch(m, torch.zeros(N, 64, 64, device=device), P)   # warm-up
+    if device == "cuda":
+        torch.cuda.synchronize()
+    print("warm-up done", flush=True)
+
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind((host, port))
+    srv.listen(4)
+    print(f"listening on {host}:{port} (N={N}, full-res shift-and-stitch, stateful)", flush=True)
+    pyr = None                                               # uploaded pyramid (per connection)
+    while True:
+        conn, addr = srv.accept()
+        conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        print(f"{datetime.now():%H:%M:%S} client {addr}", flush=True)
+        try:
+            while True:
+                cmd = struct.unpack(">i", recvall(conn, 4))[0]
+                if cmd == CMD_BYE:
+                    break
+                if cmd == CMD_UPLOAD:
+                    T, H, W = struct.unpack(">iii", recvall(conn, 12))
+                    data = recvall(conn, T * H * W * 4)
+                    log = torch.from_numpy(np.frombuffer(data, dtype=">f4").astype(np.float32)
+                                           .reshape(T, H, W)).to(device)
+                    t0 = time.perf_counter()
+                    pyr = build_pyramid(log)
+                    if device == "cuda":
+                        torch.cuda.synchronize()
+                    bms = (time.perf_counter() - t0) * 1e3
+                    nl = len(pyr)
+                    print(f"{datetime.now():%H:%M:%S} UPLOAD T={T} {H}x{W} -> {nl} levels "
+                          f"{[len(l) for l in pyr]}  build={bms:.1f}ms ({T*H*W*4/1e6:.1f}MB)", flush=True)
+                    conn.sendall(struct.pack(">i", nl) + b"".join(struct.pack(">i", len(l)) for l in pyr)
+                                 + struct.pack(">id", N, bms))   # +N so the Java client can size scene windows
+                elif cmd == CMD_INFER:
+                    level, newest, rx, ry, rw, rh = struct.unpack(">iiiiii", recvall(conn, 24))
+                    lev = pyr[level]                          # [Tl,H,W]
+                    # newest-FIRST window (channel 0 = newest), matching Java inferROI window[h]=frames[newest-h]
+                    # and the training order; .flip(0) reverses the ascending slice. By Claude on 06/20/2026
+                    win = lev[newest - N + 1:newest + 1].flip(0)  # [N,H,W]
+                    if device == "cuda":
+                        ev0 = torch.cuda.Event(enable_timing=True); ev1 = torch.cuda.Event(enable_timing=True)
+                        torch.cuda.synchronize(); ev0.record()
+                    field = shift_stitch(m, win, P)           # [C,H,W]
+                    offset5, roi_field, nvel = decode(field, vr, (rx, ry, rw, rh))
+                    if device == "cuda":
+                        ev1.record(); torch.cuda.synchronize(); gms = ev0.elapsed_time(ev1)
+                    else:
+                        gms = 0.0
+                    H, W = field.shape[1], field.shape[2]
+                    o5 = offset5.cpu().numpy().astype(">f4")          # [5,H,W]
+                    rf = roi_field.cpu().numpy().astype(">f4")        # [rh,rw,nvel]
+                    print(f"{datetime.now():%H:%M:%S} INFER lev={level} f={newest} ROI={rw}x{rh} "
+                          f"gpu_exec={gms:.1f}ms ({(o5.nbytes+rf.nbytes)/1e6:.1f}MB out)", flush=True)
+                    conn.sendall(struct.pack(">dii", gms, H, W) + o5.tobytes()
+                                 + struct.pack(">iii", rh, rw, nvel) + rf.tobytes())
+                elif cmd == CMD_READBACK:
+                    level, frame = struct.unpack(">ii", recvall(conn, 8))
+                    fr = pyr[level][frame].cpu().numpy().astype(">f4")  # [H,W]
+                    print(f"{datetime.now():%H:%M:%S} READBACK lev={level} f={frame}", flush=True)
+                    conn.sendall(struct.pack(">ii", fr.shape[0], fr.shape[1]) + fr.tobytes())
+        except (ConnectionError, struct.error, IndexError) as ex:
+            print(f"client {addr} closed/err: {ex}", flush=True)
+        finally:
+            conn.close()
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--run", default="runs/weighted9_pm_s")
+    ap.add_argument("--host", default="0.0.0.0")
+    ap.add_argument("--port", type=int, default=5577)
+    args = ap.parse_args()
+    serve(args.run, args.host, args.port)

src/main/resources/cuas_dnn/run_infer_server.sh

+#!/usr/bin/env bash
+# Start/stop the CUAS DGX inference server (PyTorch RawFCN, cuDNN) in the NGC container.
+# By Claude on 06/20/2026.  Run on the DGX (elphel@192.168.0.62).
+#   start|stop|logs|status        env: RUN=runs/<model>  PORT=5577
+set -euo pipefail
+NAME=cuas_infer
+IMG=nvcr.io/nvidia/pytorch:25.10-py3
+CODE=/home/elphel/c5p_dnn
+RUN="${RUN:-runs/weighted9_pm_s}"
+PORT="${PORT:-5577}"
+case "${1:-start}" in
+  start)
+    docker rm -f "$NAME" >/dev/null 2>&1 || true
+    docker run -d --name "$NAME" --gpus all --network host \
+      -v "$CODE":/work -w /work "$IMG" \
+      python infer_server.py --run "$RUN" --port "$PORT" >/dev/null
+    echo "started $NAME (run=$RUN port=$PORT)"; sleep 3; docker logs "$NAME"
+    ;;
+  stop)   docker rm -f "$NAME" >/dev/null 2>&1 && echo "stopped" || echo "not running" ;;
+  logs)   docker logs --tail 60 "$NAME" ;;
+  status) docker ps --filter "name=$NAME" --format "{{.Names}} {{.Status}}" ;;
+  *) echo "usage: $0 {start|stop|logs|status}  (env: RUN=, PORT=)"; exit 1 ;;
+esac