Commit a57f2dfc authored by Andrey Filippov's avatar Andrey Filippov
Browse files

Claude: refineMotionVectors() — improved debug images 



- Masked frames: replace single-frame display with full fmin..fmax stack
  (scene_titles used as slice labels; scroll to watch target stay in place)
- Source frame display removed (not needed alongside masked stack)
- Single-tile corr2d block removed; replaced by full CORR2D multi-slice image
  after the nseq loop — one slice per keyframe, same tiled layout as *CORR2D.tiff,
  slice-labeled from scene_titles, uses corr_border_contrast from CLTParameters

Co-Authored-By: default avatarClaude Sonnet 4.6 <noreply@anthropic.com>
parent 872107bf

src/main/java/com/elphel/imagej/cuas/CuasMotion.java

+426 −464
Original line number Diff line number Diff line
@@ -1560,6 +1560,15 @@ public class CuasMotion { 
		}

		return multi_targets;

	}

	public static double [][][][] readDoubleHyperHyperstack( // slices should contain two ":"

			String      path){

		return readDoubleHyperHyperstack(

				path,  // String      path,

				null,  // int []      wh,            // should be null or int[2] 

				null,  // String [][] ptop_titles,   // should be null or String [1][]

				null,  // String [][] pslice_titles, // should be null or String [1][]

				null); // int      [] err_num_ext);	}

	}

	

	public static double [][][][] readDoubleHyperHyperstack( // slices should contain two ":"

			String      path,
@@ -1577,10 +1586,13 @@ public class CuasMotion { 
			err_num[0] = 1;

			return null;

		}

		if (wh!= null) {

		if (wh == null) {

			wh = new int[2];

		}

//		if (wh!= null) {

			wh[0] = imp.getWidth();

			wh[1] = imp.getHeight();

		}

//		}

		int num_slices = imp.getStackSize();

		String [] slice_labels = imp.getStack().getSliceLabels();
@@ -1925,120 +1937,6 @@ public class CuasMotion { 
		return effective_strength;

	}

	

	public static boolean [][] filter5Targets( // should work for motion vectors and target coordinates

			final double [][][] target_sequence,

			final boolean       use_motion,     // true - use motion vectors confidence, false - use target confidence

			final boolean       select_new,     // true - use only untested tiles, false - use good tiles 

			final double        min_confidence, // 0 OK

			final double        lma_horizon, // target below horizon

			final int           tilesX,

			final int           range, // 1 or 2

			final int     []    remain,

			final int     []    passes, // debugging - number of passes required

			final int           debugLevel){

		final int conf_index = use_motion ? CuasMotionLMA.RSLT_MSCORE : CuasMotionLMA.RSLT_QSCORE; 

		final int num_seq = target_sequence.length;

		final int num_tiles = target_sequence[0].length;

		final int tilesY = num_tiles/ tilesX;

		final boolean [][] filter5 = new boolean [num_seq][num_tiles];

		final Thread[] threads = ImageDtt.newThreadArray();

		final AtomicInteger ai = new AtomicInteger(0);

		final int dbg_tile = -(38+45*80);

		final int dbg_seq0 = 13; 

		final int dbg_seq1 = 14; 

		final int ihorizon = (lma_horizon > 0) ?( (int) Math.ceil(lma_horizon/GPUTileProcessor.DTT_SIZE)) : -1; // tiles with tileY >= ihorizon are removed

		for (int ithread = 0; ithread < threads.length; ithread++) {

			threads[ithread] = new Thread() {

				public void run() {

					TileNeibs tn = new TileNeibs(tilesX, tilesY);

					boolean [] prohibit = new boolean [num_tiles];

					for (int nSeq = ai.getAndIncrement(); nSeq < num_seq; nSeq = ai.getAndIncrement()) {

						double [][] targets = target_sequence[nSeq];

						int num_total = 0;

						Arrays.fill(prohibit, false);

						for (int ntile = 0; ntile < num_tiles; ntile++) {

///							if ((targets[ntile] == null) || !(targets[ntile][conf_index] >= min_confidence) || (targets[ntile][CuasMotionLMA.RSLT_FAIL] > 0)) { // added failed

							if ((targets[ntile] == null) ||

									!(targets[ntile][conf_index] >= min_confidence) ||

									(select_new ? !Double.isNaN(targets[ntile][CuasMotionLMA.RSLT_FAIL]) : (targets[ntile][CuasMotionLMA.RSLT_FAIL] > 0))) { // defined: good or bad

								if ((ntile == dbg_tile) && (nSeq >= dbg_seq0) && (nSeq <= dbg_seq1)) {

									System.out.println("filter5Targets()):0: ntile="+ntile+", nSeq="+nSeq);

								}

								prohibit[ntile] = true;

							}

						}

						if ((ihorizon > 0) && (ihorizon < tilesY)) {

							Arrays.fill(prohibit, ihorizon*tilesX, num_tiles, true);

						}

						int num_this_pass = 0;

						int npass = 0;

						do {

							npass++;

							num_this_pass = 0;

							for (int tileY = 0; tileY < tilesY; tileY++) {

								for (int tileX = 0; tileX < tilesX; tileX++) {

									int ntile = tileX + tilesX * tileY;

									if ((ntile == dbg_tile) && (nSeq >= dbg_seq0) && (nSeq <= dbg_seq1)) {

										System.out.println("filter5Targets()):1: ntile="+ntile+", nSeq="+nSeq+", npass="+npass+", num_this_pass="+num_this_pass+", num_total="+num_total);

									}

//									if ((conf[ntile] > 0.0) && !prohibit[ntile]) {

									if (!prohibit[ntile]) { // targets[ntile][conf_index] >0 here by the prohibit preset

										boolean ismax = true;

										check_max:{

											double cval = targets[ntile][conf_index] - min_confidence;

											if (cval <= 0) {

												ismax = false;

												prohibit[ntile] = true; // not needed 

												break check_max;

											}

											for (int dy = -range; dy <= range; dy++) {

												for (int dx = -range; dx <= range; dx++) {

													int indx = tn.getNeibIndex(ntile, dx, dy);

													if ((indx >= 0) && !prohibit[indx]) {

//														double val = conf[indx]- min_confidence;

														double val = targets[indx][conf_index]- min_confidence;

														if (val > cval) {

															ismax = false;

															break check_max;

														}

													}

												}

											}

										}

										if (ismax) {

											filter5[nSeq][ntile] = true; // ismax;

											// prohibit all around, including this one

											for (int dy = -range; dy <= range; dy++) {

												for (int dx = -range; dx <= range; dx++) {

													int indx = tn.getNeibIndex(ntile, dx, dy);

													if (indx >= 0) { // && ((filtered == null) || !filtered[nSeq][indx])){

														prohibit[indx] = true;

													}

												}

											}

											num_this_pass++;

										}

									}

								}

							}

							num_total+=num_this_pass;

						} while (num_this_pass > 0);

						

						if (remain != null) {

							remain[nSeq] = num_total;

						}

						if (passes != null) {

							passes[nSeq] = npass;

						}

					}

				}

			};

		}		      

		ImageDtt.startAndJoin(threads);

		return filter5;

	}

	

	



	private static void setBadHorizon(

			final double [][][] target_sequence,
@@ -2314,7 +2212,7 @@ public class CuasMotion { 


		// Hard-coded debug selectors: set >= 0 to enable per-scan/per-tile visualisation

		final int dbg_nseq = 20; // -1;

		final int dbg_tile = 51+38*80; // -1;

		final int dbg_tile = 50+38*80; // -1;



		// Pre-compute integer-pixel raised-cosine mask kernel once (shared across all nseq)

		final int r1i = (int) Math.ceil(recalc_mv_r1);
@@ -2339,6 +2237,10 @@ public class CuasMotion { 
		final float[][] fpixels_masked = new float[nframes][width * height];

		final Thread[] threads = ImageDtt.newThreadArray();

		final AtomicInteger ai = new AtomicInteger(0);

		final boolean show_dbg = (dbg_nseq >= 0 || dbg_tile >= 0) && debugLevel >= 0;

		// Collect per-nseq correlation for end-of-method CORR2D stack

		final double[][][] corr2d_ref = show_dbg ? new double[targets_nonoverlap.length][][] : null;

		final String[] scene_titles_all = cuasMotion.getSceneTitles();



		for (int nseq = 0; nseq < targets_nonoverlap.length; nseq++) {

			// Skip scan positions where no target has a known centroid yet
@@ -2420,13 +2322,19 @@ public class CuasMotion { 
			}

			ImageDtt.startAndJoin(threads);



			// Debug: show the masked source frame at frame_center for this scan

			// Debug: show full fmin..fmax masked frame stack for dbg_nseq

			if (nseq == dbg_nseq && debugLevel >= 0) {

				int f_show = Math.max(fmin_alloc, Math.min(fmax_alloc, frame_center));

				ShowDoubleFloatArrays.showArrays(fpixels_masked[f_show].clone(), width, height,

						"refineMotionVectors-masked-nseq" + nseq + "-f" + f_show);

				ShowDoubleFloatArrays.showArrays(fpixels[f_show], width, height,

						"refineMotionVectors-source-nseq" + nseq + "-f" + f_show);

				int fcount = fmax_alloc - fmin_alloc + 1;

				float[][] stack_slices = new float[fcount][];

				String[] stack_titles = new String[fcount];

				for (int fi = 0; fi < fcount; fi++) {

					stack_slices[fi] = fpixels_masked[fmin_alloc + fi].clone();

					int f = fmin_alloc + fi;

					stack_titles[fi] = (scene_titles_all != null && f < scene_titles_all.length)

							? scene_titles_all[f] : "f" + f;

				}

				ShowDoubleFloatArrays.showArrays(stack_slices, width, height, true,

						"refineMotionVectors-masked-nseq" + nseq, stack_titles);

			}



			TDCorrTile[] tdCorrTiles = cuasMotion.correlatePairs(
@@ -2447,6 +2355,9 @@ public class CuasMotion { 
					0xFE,

					fat_zero * scale_fat_zero,

					debugLevel);

			if (corr2d_ref != null) {

				corr2d_ref[nseq] = corr_tiles_pd;

			}



			double[][] vector_field = TDCorrTile.getMismatchVector(

					corr_tiles_pd,
@@ -2455,15 +2366,6 @@ public class CuasMotion { 
					n_recenter,

					true);



			// Debug: show 2D correlation tile for the debug tile

			if (nseq == dbg_nseq && dbg_tile >= 0 && dbg_tile < corr_tiles_pd.length

					&& corr_tiles_pd[dbg_tile] != null && debugLevel >= 0) {

				int corr_side = 2 * GPUTileProcessor.DTT_SIZE - 1; // 15

				float[] corr_img = new float[corr_side * corr_side];

				for (int i = 0; i < corr_img.length; i++) corr_img[i] = (float) corr_tiles_pd[dbg_tile][i];

				ShowDoubleFloatArrays.showArrays(corr_img, corr_side, corr_side,

						"refineMotionVectors-corr2d-nseq" + nseq + "-tile" + dbg_tile);

			}



			// Add differential MV to targets_nonoverlap in-place

			for (int ntile = 0; ntile < targets_nonoverlap[nseq].length; ntile++) {
@@ -2480,6 +2382,28 @@ public class CuasMotion { 
				target[CuasMotionLMA.RSLT_VY] += vector_field[ntile][INDX_VY];

			}

		}

		// Debug: CORR2D overview — one slice per keyframe, same layout as *CORR2D.tiff

		if (show_dbg && corr2d_ref != null) {

			final int corr_size = 2 * GPUTileProcessor.DTT_SIZE - 1; // 15

			String[] slice_titles_ref = new String[targets_nonoverlap.length];

			for (int i = 0; i < targets_nonoverlap.length; i++) {

				int frame_center = frame0 + i * corr_inc;

				slice_titles_ref[i] = (scene_titles_all != null && frame_center < scene_titles_all.length)

						? scene_titles_all[frame_center] : "nseq" + i;

			}

			double[][] dbg_2d_corrs = ImageDtt.corr_partial_dbg(

					corr2d_ref,

					cuasMotion.tilesX,

					corr_size,

					clt_parameters.corr_border_contrast,

					debugLevel);

			ShowDoubleFloatArrays.makeArrays(

					dbg_2d_corrs,

					cuasMotion.tilesX * (corr_size + 1),

					cuasMotion.tilesY * (corr_size + 1),

					"refineMotionVectors-CORR2D",

					slice_titles_ref).show();

		}

	}



	/**
@@ -7921,6 +7845,9 @@ public class CuasMotion { 
				System.out.println("testCuasScanMotion(): wrong format for a pair of strength, fraction values.");

			}

		}

		boolean reuse_non_centered = true; // make a parameter

		

		

		

		double filter_below_horizon=5;

		int start_frame = 0;
@@ -7939,10 +7866,29 @@ public class CuasMotion { 
		int [] passes = new int [num_corr_samples]; // debugging filter5

		String model_prefix = parentCLT.getImageName()+getParametersSuffix(clt_parameters,null)+(cuasMotion.slow_targets? "-SLOW":"-FAST");



//		final double [][][][] target_sequence_multi = null; 

		double [][][][] target_sequence_multi = null; 

		int niter=0; // maybe start from 19 to match skipped non-center and have the same file names

		if (reuse_non_centered)	{

			String x3d_path = parentCLT.getX3dDirectory();

			String noncentered_path = x3d_path + Prefs.getFileSeparator() + model_prefix+"-ROUND_ONE"+".tiff";

			target_sequence_multi = readDoubleHyperHyperstack( // slices should contain two ":"			

							noncentered_path);           // String      path,

			if (debugLevel > -4) {

				if (target_sequence_multi!= null) {

					System.out.println("locateAndFreezeTargetsMulti(): reusing non-centered data from "+noncentered_path);

				} else {

					System.out.println("locateAndFreezeTargetsMulti(): non-centered data not found: "+noncentered_path);

					System.out.println("Will calculate and save non-centered targets");

				}

			}

//			niter = 19; //

		}

		final int num_seq = motion_sequence.length;

		final int num_tiles = motion_sequence[0].length;

		final double [][][][] target_sequence_multi = new double [num_seq][num_tiles][][]; 

		int niter=0;

		if (target_sequence_multi == null) {

			//		final double [][][][] 

			target_sequence_multi = new double [num_seq][num_tiles][][]; 

			// first pass, using non-centered targets

			//		double        boost_accum_pairs = slow_mode? 1.0:4.0; // just for testing, unconditionally boost tracking cameras exposure time

			for (; niter < num_cycles; niter++) {
@@ -7965,6 +7911,8 @@ public class CuasMotion { 


				//	double [][] effective_strength = 

				// ==== does it need to be re-calculated ? =====

				// Only used in 			int [][] filter5 = filter5Targets( // will ignore failed tiles

				// for centered tile pass, trying to eliminate with null for the motion sequence

				getEffectiveStrengthMV( // calculate tiles effective strength by the motion vectors. Combine with the target LMA?

						motion_sequence,   // final double [][][] motion_scan,

						niter,             // int                 niter, // save iteration number on failure if >=
@@ -8280,9 +8228,9 @@ public class CuasMotion { 
				}

			} //for (niter=0; niter < max_iter; niter++)



		if (intermed_low) {

			if (intermed_low || reuse_non_centered) {

				ImagePlus imp_new_scores = showTargetSequence(

					target_sequence_multi,          // double [][][] vector_fields_sequence,

						target_sequence_multi,          // double [][][][] vector_fields_sequence,

						slice_titles,         // String []     titles, // all slices*frames titles or just slice titles or null

						model_prefix+"-ROUND_ONE",// String        title,

						false, // good_only,            // final boolean         good_only,
@@ -8291,7 +8239,7 @@ public class CuasMotion { 
						cuasMotion.tilesX);   //  int           tilesX) {

				parentCLT.saveImagePlusInModelDirectory(imp_new_scores);  // ImagePlus   imp)

			}

		

		}

		



		//===========================================================================
@@ -8329,7 +8277,7 @@ public class CuasMotion { 
			}



			int [][] filter5 = filter5Targets( // will ignore failed tiles

					motion_sequence,       // final double [][][] target_sequence,

					null, // motion_sequence,       // final double [][][] target_sequence,

					target_sequence_multi, // final double [][][][] target_sequence_multi,non-centered marked as "used"

					// if     use motion and select_new will only consider tiles (and compare motion scores in motion_sequence) that have nulls in target_sequence_multi[nseq][ntile]

					// if not use motion and select_new will only consider (and compare scores) tiles that have [RSLT_CENTERED] = 0.0
@@ -8341,7 +8289,7 @@ public class CuasMotion { 
					cuasMotion.tilesX,  // final int           tilesX,

					max_range,          // final int           range, // 1 or 2

					filter5_remain,     // final int     []    remain){

					passes,             // final int     []    passes, // debugging - numer of passes required

					passes,             // final int     []    passes, // debugging - number of passes required

					debugLevel);        // final int           debugLevel)



			if (save_filtered_low) {
@@ -8413,7 +8361,7 @@ public class CuasMotion { 
				// show good and bad accumulated here too?

			}



			// Andrey 05/05/2026 moved here (earlier) so shiftAndRenderAccumulate() will use update motion vector

			// Andrey 05/05/2026 moved here (earlier) so shiftAndRenderAccumulate() will use updated motion vector

			// By Claude on 05/05/2026 — re-correlate with spatial mask around known target

			if (recalc_mv) {

				refineMotionVectors(
@@ -8669,7 +8617,21 @@ public class CuasMotion { 
		}

		return target_sequence_multi; // target_sequence; // contains all tiles, good or bad, their data and when they were defined

	}

	

	/**

	 * 		if (select_new && !use_motion) , fill mark all selected non-centered as "used"

	 * @param motion_sequence

	 * @param target_sequence_multi

	 * @param use_motion_in

	 * @param select_new_in

	 * @param min_confidence

	 * @param lma_horizon

	 * @param tilesX

	 * @param range

	 * @param remain

	 * @param passes

	 * @param debugLevel

	 * @return

	 */

	public static int [][] filter5Targets( // should work for motion vectors and target coordinates returns index of selected target or -1 for unselected

			final double [][][]   motion_sequence, // may be null

			final double [][][][] target_sequence_multi, // not null
@@ -8688,16 +8650,16 @@ public class CuasMotion { 
			final int             debugLevel){

		final boolean  use_motion = (motion_sequence != null) ? use_motion_in : false;

		final boolean  select_new = (motion_sequence != null) ? select_new_in : false;

		boolean debug_now = (debugLevel>5);

		boolean debug_now = (debugLevel<5);

		final int num_seq = target_sequence_multi.length;

		final int num_tiles = target_sequence_multi[0].length;

		final int tilesY = num_tiles/ tilesX;

		final int [][] filter5 = new int [num_seq][num_tiles];

		final Thread[] threads = ImageDtt.newThreadArray();

		final AtomicInteger ai = new AtomicInteger(0);

		final int dbg_tile = debug_now ?(53+38*80) : -1;

		final int dbg_seq0 = debug_now ? 47: -1; 

		final int dbg_seq1 = debug_now ? 49: -1; 

		final int dbg_tile = debug_now ?(50+38*80) : -1;

		final int dbg_seq0 = debug_now ? 20: -1; 

		final int dbg_seq1 = debug_now ? 21: -1; 

		final int ihorizon = (lma_horizon > 0) ?( (int) Math.ceil(lma_horizon/GPUTileProcessor.DTT_SIZE)) : -1; // tiles with tileY >= ihorizon are removed

		if (dbg_tile >=0) {

			System.out.println("filter5Targets()):00: use_motion="+use_motion+", select_new="+select_new+", ihorizon="+ihorizon);