Before implementing tree-top filter

src/main/java/com/elphel/imagej/tileprocessor/IntersceneMatchParameters.java

@@ -859,6 +859,7 @@ min_str_neib_fpn	0.35
 	
 	
 	// combine  parameters
+	public boolean   terr_crop =               true; // Crop output image to the bounds of all segments
 	public int       terr_border_width =       6;
 	public boolean   terr_render_open =        true; // render open areas (no vegetation offset)
 	public boolean   terr_render_no_alpha =    true; // render where no opacity is available
@@ -876,6 +877,8 @@ min_str_neib_fpn	0.35
 	public double    terr_min_terrain =        0.001;
 	public double    terr_min_vegetation =     0.5;
 	public int       terr_debug =              0;
+	public String    terr_debug_path =        "/media/elphel/SSD3-4GB/lwir16-proc/berdich3/debug/vegetation/lma_um/";
+	public int       terr_debug_length =       200;
 	public boolean   terr_debug_improved =     true; // Save debug data if LMA step improved
 	public boolean   terr_debug_worsened =     true; // Save debug data if LMA step improved
 	
@@ -2191,6 +2194,7 @@ min_str_neib_fpn	0.35
 		
 		
 		gd.addMessage     ("Combining LMA results segments");
+		gd.addCheckbox    ("Crop result image",    terr_crop,                   "Crop output image to the bounds of all segments.");
 		gd.addNumericField("Overlap width",        terr_border_width, 0,3,"","Width of the inter-tile oiverlap border.");
 		gd.addCheckbox    ("Render open",          terr_render_open, "");
 		gd.addCheckbox    ("Render no alpha",      terr_render_no_alpha, "");
@@ -2206,7 +2210,9 @@ min_str_neib_fpn	0.35
 		gd.addNumericField("Min max terrain",      terr_min_max_terrain, 5,7,"", 			".");
 		gd.addNumericField("Minimal terrain",      terr_min_terrain, 5,7,"", 			".");
 		gd.addNumericField("Minimal vegetation",   terr_min_vegetation, 5,7,"", 			".");
-		gd.addNumericField("LMA debug level",      terr_debug, 0,3,"", 			".");
+		gd.addNumericField("LMA debug level",      terr_debug, 0,3,"", "Debug level.");
+		gd.addStringField ("Debug directory",      terr_debug_path, 120,"Directory to save debug images.");
+		gd.addNumericField("Debug length",         terr_debug_length, 0,3,"","Maximal number of debug image slices.");
 		gd.addCheckbox    ("Debug improved",       terr_debug_improved, "Save debug image after successful LMA step.");
 		gd.addCheckbox    ("Debug worsened",       terr_debug_worsened, "Save debug image after unsuccessful LMA step.");
 		
@@ -2939,6 +2945,7 @@ min_str_neib_fpn	0.35
 		terr_recalc_boost =        gd.getNextNumber(); // double 
 		terr_recalc_average =      gd.getNextBoolean();// boolean
 		
+		terr_crop =                gd.getNextBoolean();// boolean
 		terr_border_width =  (int) gd.getNextNumber(); // int    
 		terr_render_open =         gd.getNextBoolean();// boolean
 		terr_render_no_alpha =     gd.getNextBoolean();// boolean
@@ -2953,7 +2960,9 @@ min_str_neib_fpn	0.35
 		terr_min_max_terrain =     gd.getNextNumber();// double 
 		terr_min_terrain =         gd.getNextNumber();// double 
 		terr_min_vegetation =      gd.getNextNumber();// double 
-		terr_debug =         (int) gd.getNextNumber(); // int    
+		terr_debug =         (int) gd.getNextNumber(); // int
+		terr_debug_path =          gd.getNextString(); // String
+		terr_debug_length =  (int) gd.getNextNumber(); // int
 		terr_debug_improved =      gd.getNextBoolean();// boolean
 		terr_debug_worsened =      gd.getNextBoolean();// boolean
 // end of vegetation		
@@ -3663,6 +3672,7 @@ min_str_neib_fpn	0.35
 		properties.setProperty(prefix+"terr_recalc_boost",           terr_recalc_boost+"");       // double 
 		properties.setProperty(prefix+"terr_recalc_average",         terr_recalc_average+"");     // boolean
 
+		properties.setProperty(prefix+"terr_crop",                   terr_crop+"");               // boolean
 		properties.setProperty(prefix+"terr_border_width",           terr_border_width+"");       // int    
 		properties.setProperty(prefix+"terr_render_open",            terr_render_open+"");        // boolean
 		properties.setProperty(prefix+"terr_render_no_alpha",        terr_render_no_alpha+"");    // boolean
@@ -3677,7 +3687,9 @@ min_str_neib_fpn	0.35
 		properties.setProperty(prefix+"terr_min_max_terrain",        terr_min_max_terrain+"");    // double 
 		properties.setProperty(prefix+"terr_min_terrain",            terr_min_terrain+"");        // double 
 		properties.setProperty(prefix+"terr_min_vegetation",         terr_min_vegetation+"");     // double 
-		properties.setProperty(prefix+"terr_debug",                  terr_debug+"");              // int    
+		properties.setProperty(prefix+"terr_debug",                  terr_debug+"");              // int
+		properties.setProperty(prefix+"terr_debug_path",             terr_debug_path+"");         // String
+		properties.setProperty(prefix+"terr_debug_length",           terr_debug_length+"");       // int
 		properties.setProperty(prefix+"terr_debug_improved",         terr_debug_improved+"");     // boolean
 		properties.setProperty(prefix+"terr_debug_worsened",         terr_debug_worsened+"");     // boolean
 		// end of vegetation
@@ -4411,6 +4423,7 @@ min_str_neib_fpn	0.35
 		if (properties.getProperty(prefix+"terr_recalc_boost")!=       null) terr_recalc_boost=Double.parseDouble(properties.getProperty(prefix+"terr_recalc_boost"));
 		if (properties.getProperty(prefix+"terr_recalc_average")!=     null) terr_recalc_average=Boolean.parseBoolean(properties.getProperty(prefix+"terr_recalc_average"));
 		
+		if (properties.getProperty(prefix+"terr_crop")!=               null) terr_crop=Boolean.parseBoolean(properties.getProperty(prefix+"terr_crop"));
 		if (properties.getProperty(prefix+"terr_border_width")!=       null) terr_border_width=Integer.parseInt(properties.getProperty(prefix+"terr_border_width"));
 		if (properties.getProperty(prefix+"terr_render_open")!=        null) terr_render_open=Boolean.parseBoolean(properties.getProperty(prefix+"terr_render_open"));
 		if (properties.getProperty(prefix+"terr_render_no_alpha")!=    null) terr_render_no_alpha=Boolean.parseBoolean(properties.getProperty(prefix+"terr_render_no_alpha"));
@@ -4426,6 +4439,8 @@ min_str_neib_fpn	0.35
 		if (properties.getProperty(prefix+"terr_min_terrain")!=        null) terr_min_terrain=Double.parseDouble(properties.getProperty(prefix+"terr_min_terrain"));
 		if (properties.getProperty(prefix+"terr_min_vegetation")!=     null) terr_min_vegetation=Double.parseDouble(properties.getProperty(prefix+"terr_min_vegetation"));
 		if (properties.getProperty(prefix+"terr_debug")!=              null) terr_debug=Integer.parseInt(properties.getProperty(prefix+"terr_debug"));
+		if (properties.getProperty(prefix+"terr_debug_path")!=         null) terr_debug_path=(String) properties.getProperty(prefix+"terr_debug_path");
+		if (properties.getProperty(prefix+"terr_debug_length")!=       null) terr_debug_length=Integer.parseInt(properties.getProperty(prefix+"terr_debug_length"));
 		if (properties.getProperty(prefix+"terr_debug_improved")!=     null) terr_debug_improved=Boolean.parseBoolean(properties.getProperty(prefix+"terr_debug_improved"));
 		if (properties.getProperty(prefix+"terr_debug_worsened")!=     null) terr_debug_worsened=Boolean.parseBoolean(properties.getProperty(prefix+"terr_debug_worsened"));
 		// end of vegetation
@@ -5119,6 +5134,7 @@ min_str_neib_fpn	0.35
 		imp.terr_recalc_boost =                  this.terr_recalc_boost;
 		imp.terr_recalc_average =                this.terr_recalc_average;
 		
+		imp.terr_crop =                          this.terr_crop;
 		imp.terr_border_width =                  this.terr_border_width;                           
 		imp.terr_render_open =                   this.terr_render_open;                           
 		imp.terr_render_no_alpha =               this.terr_render_no_alpha;                           
@@ -5134,7 +5150,8 @@ min_str_neib_fpn	0.35
 		imp.terr_min_terrain =                   this.terr_min_terrain;                           
 		imp.terr_min_vegetation =                this.terr_min_vegetation;  
 		imp.terr_debug =                         this.terr_debug;
-		
+		imp.terr_debug_path =                    this.terr_debug_path;
+		imp.terr_debug_length =                  this.terr_debug_length;
 		imp.terr_debug_improved =                this.terr_debug_improved;
 		imp.terr_debug_worsened =                this.terr_debug_worsened;
 		// end of vegetation

src/main/java/com/elphel/imagej/vegetation/VegetationModel.java

@@ -1596,14 +1596,21 @@ public class VegetationModel {
 		double    transparency_pow =      clt_parameters.imp.terr_recalc_pwr ;          //  1.0; // Raise transparency to this power when calculating weight  
 		double    transparency_gb =       clt_parameters.imp.terr_recalc_gb ;           //  2.0; // Blur transparency-based confidence  
 		double    transparency_boost =    clt_parameters.imp.terr_recalc_boost ;        //  5.0; // Maximal boost while increasing low-confidence pixel weights  
-		boolean   recalc_average =        clt_parameters.imp.terr_recalc_average ;      //false; // apply transparency to average mismatch
+		boolean   recalc_average =        clt_parameters.imp.terr_recalc_average ;      // false; // apply transparency to average mismatch
+		String 	  debug_path =            clt_parameters.imp.terr_debug_path;           // Directory to save debug images
 		boolean   debug_save_improved =   clt_parameters.imp.terr_debug_improved;
 		boolean   debug_save_worsened =   clt_parameters.imp.terr_debug_worsened;
+		int       debug_length =          clt_parameters.imp.terr_debug_length;
+		
+		
+		
+		
 //		boolean restore_mode =   false;
 		boolean save_par_files = true; // false;
 		String segments_dir = getSegmentsDir(segments_sub);
 		boolean   read_pars =     false; // true; /// false; /// true; //   false; // true;
-		
+		boolean   crop_combo =   clt_parameters.imp.terr_crop; // Crop output image to the bounds of all segments
+
 		// debug feature to read to continue - needs to be cleaned up/replaced
 //		String  parameters_path = "/media/elphel/SSD3-4GB/lwir16-proc/berdich3/debug/vegetation/essential/parameters_vector_data_x143-y317-w35-h35-al100.0-alo0.0-alp10.0-alin-tl0.2-vl0.01-tp0.01-vp0.01-bp5.0-um1.0_0.8.tiff";
 
@@ -2061,7 +2068,6 @@ public class VegetationModel {
 					segments_dir,     // String dir_path)
 					segments_suffix); // String suffix);
 			*/
-			boolean crop_combo = true;
 			VegetationSegment [] segments = vegetationLMA.readAllSegments(
 					segments_dir,          // String dir_path)
 					segments_suffix,       // String suffix);
@@ -2072,6 +2078,7 @@ public class VegetationModel {
 					transparency_pow,      // double        transparency_pow,			
 					transparency_gb,       // double        transparency_gb,
 					debugLevel,            // int           debugLevel,
+					debug_path,            // String        debug_path,
 					debug_save_improved,   // boolean       debug_save_improved, // Save debug image after successful LMA step."); 
 					debug_save_worsened);  // boolean       debug_save_worsened) { // Save debug image after unsuccessful LMA step.");
 			VegetationSegment.combineSegments(
@@ -2116,9 +2123,11 @@ public class VegetationModel {
 			int num_samples = vegetationLMA.prepareLMA(
 					false,             // final boolean            keep_parameters,
 					woi,               // final Rectangle   woi,
+					null, // final Rectangle          woi_veg_in, // used when loading from file (may be different)
 					max_warp,          //			final double             max_warp, // 1.8 - do not use scenes where distance between vegetation projection exceeds this
 					max_offset,        // final int                max_offset,               // maximal "elevation" to consider 					
 					elevation_radius,  // final double             elevation_radius, // Radius of elevation/vegetation influence.
+					null, // final boolean []         valid_scene_pix,         
 					hifreq_weight,     //final double             hifreq_weight,  // 22.5 0 - do not use high-freq. Relative weight of laplacian components
 					terrain_correction,// final double             terrain_correction,
 					fit_terr,          // final boolean            adjust_terr,
@@ -2158,8 +2167,9 @@ public class VegetationModel {
 					max_parallax,   //final double             max_parallax,  // do not consider maximal parallax above this (consider it a glitch) 
 //					um_sigma,      // final double             um_sigma,           // just use in debug image names
 //					(um_en? um_weight: 0.0),     // final double             um_weight,
-//					par_path,       // final String             parameters_read_path, 
-					debugLevel,   // final int         debugLevel);
+//					par_path,            // final String             parameters_read_path, 
+					debugLevel,          // final int         debugLevel);
+					debug_path,          // final String             debug_path,
 					debug_save_improved, // final boolean            debug_save_improved, // Save debug image after successful LMA step."); 
 					debug_save_worsened);// final boolean            debug_save_worsened) // Save debug image after unsuccessful LMA step.");
 
@@ -2168,17 +2178,38 @@ public class VegetationModel {
 				continue;
 			}
 			
+//			if (save_par_files && skip_existing_woi) { // check that segment already exists
 			if (save_par_files && skip_existing_woi) { // check that segment already exists
+				//	public boolean segmentFileExist(String dir_path) {
+/*
 				String save_path = VegetationLMA.getSavePath(
 						segments_dir, // String dir,
 						vegetationLMA.getParametersDebugTitle()); // String title)
 				if (new File (save_path).exists()) {
-					System.out.println("File "+save_path+"\n already exists, skipping this woi");
-					continue;
-				}
-				if (new File (save_path.replace("-new-","-file-")).exists()) {
-					System.out.println("File "+save_path+"\n already exists, skipping this woi");
-					continue;
+				*/
+				File[] segment_files = vegetationLMA.getSegmentFiles(segments_dir);
+				if (tile_woi) {
+					if (segment_files.length > 0) {
+						for (File f : segment_files) {
+							System.out.println("File "+f+"\n with the same woi already exists, skipping this woi");
+						}
+						continue;
+					}
+				} else {
+					String save_path = VegetationLMA.getSavePath(
+							segments_dir, // String dir,
+							vegetationLMA.getParametersDebugTitle()); // String title)
+					if (new File (save_path).exists()) {
+						System.out.println("File "+save_path+"\n (exactly) already exists, skipping this woi");
+//						for (File f : segment_files) {
+//							System.out.println("File "+f+"\n (exactly) already exists, skipping this woi");
+//						}
+						continue;
+					}
+					if (new File (save_path.replace("-new-","-file-")).exists()) {
+						System.out.println("File "+save_path+"\n already exists, skipping this woi");
+						continue;
+					}
 				}
 			}
 			int num_iter = num_iters[step_restore]; //
@@ -2250,7 +2281,8 @@ public class VegetationModel {
 			}
 			/// next_run = true;
 			vegetationLMA.debug_index = 0;
-			vegetationLMA.debug_image = new double [100][]; // num_iter][];
+			vegetationLMA.debug_length = debug_length;
+			vegetationLMA.debug_image = new double [vegetationLMA.debug_length][]; // num_iter][];
 
 			int lma_rslt= vegetationLMA.runLma( // <0 - failed, >=0 iteration number (1 - immediately)
 					lambda,           // double lambda,           // 0.1

src/main/java/com/elphel/imagej/vegetation/VegetationSegment.java

@@ -15,6 +15,7 @@ public class VegetationSegment {
 	public double [][] tva; // woi_veg
 	public double      terrain_offset;
 	public double []   confidence; // woi
+	public boolean []  overlaid;
 	public String      path;
 	public VegetationSegment(
 			String      path, 
@@ -23,7 +24,8 @@ public class VegetationSegment {
 			double terrain_offset,
 			double []   scene_offsets, // has NaNs
 			double [][]  tva,
-			double []   confidence) {
+			double []   confidence,
+			boolean []  overlaid) {
 		this.path = path;
 		this.woi = woi;
 		this.woi_veg = woi_veg;
@@ -31,6 +33,22 @@ public class VegetationSegment {
 		this.tva =  tva;
 		this.terrain_offset = terrain_offset;
 		this.confidence = confidence;
+		this.overlaid = overlaid;
+		correctTerrainOffset();
+	}
+	
+	public void correctTerrainOffset() {
+		if (!Double.isNaN(terrain_offset) && (overlaid != null)) {
+			int woi_length = woi.width * woi.height;
+			for (int windx = 0; windx < woi_length; windx++) {
+				int wx = windx % woi.width;
+				int wy = windx / woi.height;
+				int wvindx = (wx + woi.x - woi_veg.x) + (wy + woi.y - woi_veg.y) * woi_veg.width;
+				if (overlaid[windx]) {
+					tva[0][wvindx] += terrain_offset;
+				}
+			}
+		}
 	}
 	
 	public static Rectangle getBounds( // [0] - woi, [1] - woi_weg
@@ -98,7 +116,7 @@ public class VegetationSegment {
 						int x = wx +  woi.x - out_woi.x;
 						int y = wy +  woi.y - out_woi.y;
 						int indx = x + y * out_woi.width;
-						preview_data[t][ns][indx] = woi_tva[t][wsrc];
+						preview_data[t][ns][indx] = woi_tva[t][wsrc]; // java.lang.ArrayIndexOutOfBoundsException: Index -43 out of bounds for length 550
 					}
 					break;
 				case 1: // vegetation
@@ -156,13 +174,14 @@ public class VegetationSegment {
 		}
 		VegetationModel vegetationModel = vegetationLMA.getModel();
 		String ref_scene = vegetationModel.reference_scene;
+		String pre_title = ref_scene+"-preview_segments-"+out_woi.x+"-"+out_woi.y+"-"+out_woi.width+"-"+out_woi.height;
 		ShowDoubleFloatArrays.showArraysHyperstack(
-				preview_data,              // double[][][] pixels, 
-				out_woi.width,                // int          width, 
-				ref_scene+"-preview_segments",        // String       title, "time_derivs-rt"+diff_time_rt+"-rxy"+diff_time_rxy,
-				titles,                    // String []    titles, // all slices*frames titles or just slice titles or null
-				top_titles,                // String []    frame_titles, // frame titles or null
-				true);                     // boolean      show)
+				preview_data,     // double[][][] pixels, 
+				out_woi.width,    // int          width, 
+				pre_title,        // String       title, "time_derivs-rt"+diff_time_rt+"-rxy"+diff_time_rxy,
+				titles,           // String []    titles, // all slices*frames titles or just slice titles or null
+				top_titles,       // String []    frame_titles, // frame titles or null
+				true);            // boolean      show)
 		
 		double [] terrain_reference =   crop(
 				vegetationModel.terrain_scenes_render[vegetationModel.reference_index],
@@ -185,13 +204,23 @@ public class VegetationSegment {
 				um_weight);    // final double    um_weight)
 		String [] result_titles = {"ref_render", "terrain_accum",String.format("terrain_accum_UM%5.2f-%4.2f",um_sigma,um_weight),"terrain"};
 		double [][] result_img = {terrain_reference,terrain_accumulated,terrain_um,preview_data[0][accum_indx]};
-		// ref_scene+"-result-terrain"
+		boolean normalize = true;
+		int norm_index = 1;
+		if (normalize) {
+		result_img =  normalize (
+				result_img, // final double [][] data,
+				norm_index); // final int         norm_index){
+		}
+		String res_title = ref_scene+"-result_terrain-"+out_woi.x+"-"+out_woi.y+"-"+out_woi.width+"-"+out_woi.height;
+		if (normalize) {
+			res_title+= "-normalized"+norm_index;
+		}
 		ShowDoubleFloatArrays.showArrays(
 				result_img,
 				out_woi.width,
 				out_woi.height,
 				true,
-				ref_scene+"-result_terrain",
+				res_title,
 				result_titles);
 
 		
@@ -439,6 +468,68 @@ public class VegetationSegment {
 		}
 		return cropped;
 	}
-
+	/**
+	 * Normalize slices to match the selected one by average and standard deviation
+	 * @param data       array of image slices
+	 * @param norm_index index (0-based) of the reference slice
+	 * @return normalized image
+	 */
+	public static double [][] normalize (
+			final double [][] data,
+			final int         norm_index){
+		final int num_slices = data.length;
+		final int num_pix = data[0].length; 
+		double [][] normalized = new double [num_slices][];
+		for (int i = 0; i < num_slices; i++) {
+			normalized[i] = data[i].clone();
+		}
+		final double [] mean = new double[num_slices];
+		final double [] std = new double[num_slices];
+		final Thread[] threads = ImageDtt.newThreadArray(QuadCLT.THREADS_MAX);
+		final AtomicInteger ai = new AtomicInteger(0);
+		for (int ithread = 0; ithread < threads.length; ithread++) { // first sum for pairs
+			threads[ithread] = new Thread() {
+				public void run() {
+					for (int nSlice = ai.getAndIncrement(); nSlice < num_slices; nSlice = ai.getAndIncrement()) {
+						double [] slice = normalized[nSlice];
+						double s0=0, s1=0, s2=0;
+						for (int i = 0; i < slice.length; i++) if (!Double.isNaN(slice[i])) {
+							s0 += 1;
+							s1 += slice[i];
+							s2 += slice[i]*slice[i];
+						}
+						s1 /= s0;
+						s2 /= s0;
+						mean[nSlice] = s1;
+						std[nSlice] = Math.sqrt(s2 - s1 * s1);
+					}
+				}
+			};
+		}		      
+		ImageDtt.startAndJoin(threads);
+		final double [] a = new double [num_slices], b = new double[num_slices];
+		Arrays.fill(a, 1);
+		for (int n = 0; n < num_slices; n++) if (n != norm_index){
+			a[n] = std[norm_index]/std[n];
+			b[n] = mean[norm_index] - a[n]*mean[n]; 
+		}
+		ai.set(0);
+		for (int ithread = 0; ithread < threads.length; ithread++) { // first sum for pairs
+			threads[ithread] = new Thread() {
+				public void run() {
+					for (int nPix = ai.getAndIncrement(); nPix < num_pix; nPix = ai.getAndIncrement()) {
+						for (int n = 0; n < num_slices; n++) if (n != norm_index){
+							double d = normalized[n][nPix];
+							if (!Double.isNaN(d)) {
+								normalized[n][nPix] = a[n]*d + b[n]; 
+							}
+						}						
+					}
+				}
+			};
+		}		      
+		ImageDtt.startAndJoin(threads);
+		return normalized;
+	}
 	
 }