continue refactoring, migrating to more flexible ortho code

src/main/java/com/elphel/imagej/orthomosaic/ComboMatch.java

@@ -53,6 +53,7 @@ public class ComboMatch {
 	public static int gpu_max_width=  4096;
 	public static int gpu_max_height= 4096;
 	public static String [] FILES_LISTS_PATHS = {
+			"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/nov3_50-75-orange",
 			"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/nov3_50-75",
 			"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/maps_sep12-13_50-25-50-75-100m",
 			"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/maps_19_sep13_25-50-75-100m",
@@ -176,8 +177,8 @@ public class ComboMatch {
 		
 		
 		
-		boolean use_saved_collection = true; // false;
-		boolean save_collection =      true;
+		boolean use_saved_collection = false; // true; // false;
+		boolean save_collection =      false; // true;
 		boolean process_correlation = true; // use false to save new version of data
 		int     num_tries_fit =       10;
 		boolean update_match =        true; // use false to save new version of data
@@ -190,6 +191,7 @@ public class ComboMatch {
 		boolean restore_temp =        true;
 		double  frac_remove  =        clt_parameters.imp.pmtch_frac_remove; //   0.15;
 		double  metric_error =        clt_parameters.imp.pmtch_metric_err; // 0.05; // 0.02;//  2 cm
+		boolean update_files =        false;
 		boolean update_lla =          false; // re-read file metadata
 		boolean update_kernel_patterns = false;
 		boolean update_bl_bc =        false;
@@ -248,6 +250,7 @@ public class ComboMatch {
 		if (use_marked_image ) {
 			gd.addCheckbox    ("Use marked image data",   true, "Use markes from the selected image");
 		}
+		gd.addCheckbox    ("Update scene files",        update_files, "Re-read files as specified in .list file, possibly changing versions");
 		gd.addCheckbox    ("Update files metadata",     update_lla, "Re-read files metadata (if it was modified)");
 		gd.addCheckbox    ("Update kernels/patterns",   update_kernel_patterns, "Re-read kernels and patterns from *.list file");
 		gd.addCheckbox    ("Remove duplicate scenes",   fix_duplicates, "Remove scenes with the same timestamp");
@@ -304,6 +307,7 @@ public class ComboMatch {
 		if (use_marked_image ) { // will only be used if found and asked
 			use_marked_image=       gd.getNextBoolean();
 		}
+		update_files=               gd.getNextBoolean();
 		update_lla=                 gd.getNextBoolean();
 		update_kernel_patterns=     gd.getNextBoolean();
 		fix_duplicates=             gd.getNextBoolean();
@@ -321,8 +325,6 @@ public class ComboMatch {
 			} catch (ClassNotFoundException | IOException e) {
 				use_files_list = true;
 				System.out.println("Saved data not found: "+orthoMapsCollection_path+", building from a list "+files_list_path);
-				// TODO Auto-generated catch block
-//				e.printStackTrace();
 			}
 			//files_list_path
 			if (!use_files_list) {
@@ -330,12 +332,15 @@ public class ComboMatch {
 					maps_collection.updateKernels(files_list_path);
 					maps_collection.updatePatterns(files_list_path);
 				}
+				if (update_files) {
+					maps_collection.updateFiles(files_list_path);
+				}
 			}
 		} else {
 			use_files_list = true;
 		}
 		if (use_files_list) { // using files list
-			maps_collection = new OrthoMapsCollection(files_list_path); // should have ".list" extension
+			maps_collection = new OrthoMapsCollection(files_list_path, null); // should have ".list" extension
 			maps_collection.updateNumberScenes();
 			maps_collection.updateSfmGain();
 		}

src/main/java/com/elphel/imagej/orthomosaic/ModelRegex.java

+package com.elphel.imagej.orthomosaic;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.nio.file.attribute.FileTime;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.regex.Pattern;
+
+import ij.Prefs;
+
+public class ModelRegex implements Serializable{
+	private static final long serialVersionUID = 1L;	
+	public String name;
+	public String regex;
+	public LocalDateTime start_date;
+	public LocalDateTime end_date;
+
+	public ModelRegex(
+			String        name,
+			String        regex,
+			LocalDateTime start_date,
+			LocalDateTime end_date) {
+		this.name =       name;
+		this.regex =      regex;
+		this.start_date = start_date;
+		this.end_date =   end_date;
+	}
+	
+	private void writeObject(ObjectOutputStream oos) throws IOException {
+		oos.defaultWriteObject();
+	}
+	private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
+		ois.defaultReadObject();
+	}
+	
+	public String replaceByFilter (String path) {
+		String model_path = OrthoMap.getModelPathFromPath(path);
+		File model_dir = (new File(model_path)); // .getParentFile();
+		if (!model_dir.exists() || !model_dir.isDirectory()) {
+			return null;
+		}
+		File [] files = model_dir.listFiles();
+		boolean [] matches = new boolean [files.length];
+		int num_matches = 0;
+		Pattern filename_pattern= Pattern.compile(regex);
+		int latest=-1;
+		LocalDateTime [] dts = new LocalDateTime[files.length];
+		for (int i = 0; i < files.length; i++) if (files[i].isFile()){
+			long fileLastModifiedDate = files[i].lastModified();
+			dts[i] = Instant.ofEpochMilli(fileLastModifiedDate).atZone(ZoneId.systemDefault()).toLocalDateTime();
+			if ((start_date != null) && dts[i].isBefore(start_date)){
+				continue; // too early
+			}
+			if ((end_date != null) && dts[i].isAfter(end_date)){
+				continue; // too late
+			}
+			if (!filename_pattern.matcher(files[i].getName()).matches()) {
+				continue; // does not match
+			}
+			matches[i] = true;
+			num_matches++;
+			if ((latest < 0) || dts[i].isAfter(dts[latest])) {
+				latest = i;
+			}
+		}
+		if (num_matches >1) {
+			System.out.println("dir="+model_dir+", num_matches="+num_matches);
+		}
+		if (latest >= 0) {
+			//Prefs.getFileSeparator()
+			return model_path+Prefs.getFileSeparator()+files[latest].getName();
+		}
+		// find latest
+		return null;
+	}
+	
+	public static String [] removeDuplicatesSort(String [] paths) {
+		HashMap<String,String> scene_files = new HashMap<String,String>();
+		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss.SS zzz");// may be VV instead of zzz
+		for (int i = 0; i < paths.length; i++) {
+			
+			String sts = OrthoMap.getNameFromPath(paths[i]);
+			String model_dir =  OrthoMap.getModelPathFromPath(paths[i]);
+			if (scene_files.containsKey(sts)) {
+				String path_old = scene_files.get(sts);
+				long old_date = (new File(path_old)).lastModified();
+				long new_date = (new File(paths[i])).lastModified();
+				boolean replace = new_date > old_date; 
+				LocalDateTime old_dt=Instant.ofEpochMilli(old_date).atZone(ZoneId.systemDefault()).toLocalDateTime();
+				LocalDateTime new_dt=Instant.ofEpochMilli(new_date).atZone(ZoneId.systemDefault()).toLocalDateTime();
+				System.out.println("Duplicate scene found: "+sts+" in two files:");
+				System.out.println("1: "+path_old+" @ "+old_dt.atZone(ZoneId.systemDefault()).format(formatter));
+				System.out.println("2: "+paths[i]+" @ "+new_dt.atZone(ZoneId.systemDefault()).format(formatter));
+				System.out.println("Using "+(replace ? 2 : 1) +" as it is not older");
+				if (replace) {
+					scene_files.put(sts, paths[i]);
+				}
+			} else {
+				scene_files.put(sts, paths[i]);
+			}
+		}
+		// assuming string ts have the same order as double timestamps
+		String [] filtered_ts = scene_files.keySet().toArray(new String[0]);
+		Arrays.sort(filtered_ts);
+		String [] filtered_paths = new String [filtered_ts.length];
+		for (int i = 0; i < filtered_paths.length; i++) {
+			filtered_paths[i] = scene_files.get(filtered_ts[i]);
+		}
+		return filtered_paths;
+	}
+	
+	
+	
+}

src/main/java/com/elphel/imagej/orthomosaic/OrthoMap.java

@@ -588,17 +588,22 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
 	 */
 	public FloatImageData getAlt() {
 		if (alt_image == null) {
-			String alt_path = getAltPath(path);
+			alt_image = getAlt(path); // img path, not alt
+		}
+		return alt_image;
+	}
+	
+	public static FloatImageData getAlt(String img_path) {
+		String alt_path = getAltPath(img_path);
 		File alt_file = new File(alt_path);
 		if (!alt_file.exists()) {
 			System.out.println("Altitudes file does not exist: "+alt_path);
 			return null;
 		}
-			alt_image = new FloatImageData(alt_path);
-		}
-		return orig_image;
+		return new FloatImageData(alt_path);
 	}
 
+	
 	/*
 	public OrthoMap (
 			String path) {
@@ -676,6 +681,11 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
 	public void setAffine(double [][] affine) {
 		this.affine = new double[][] {affine[0].clone(),affine[1].clone()};
 	}
+
+	public void setAffine() {
+		this.affine = new double[][] {{1,0,0},{0,1,0}};
+	}
+	
 	public double [][] getAffine(){
 		return affine;
 	}

src/main/java/com/elphel/imagej/orthomosaic/OrthoMapsCollection.java

@@ -86,10 +86,13 @@ public class OrthoMapsCollection  implements Serializable{
 	public static final String [] NAME_MO = {"main","other"};
 	public transient long                       version = -1;
 	OrthoMap []                                 ortho_maps;
+	// remove some transients
 	transient HashMap<Double,Integer>           map_index;
 	transient HashMap<String,Integer>           map_index_string;
 	transient ArrayList<AltitudeMismatchKernel> kernels;
 	transient ArrayList<GroundObjectPattern>    patterns;
+	transient HashMap<String,ModelRegex>        model_regexes;
+	transient String                            current_model_regex="";
 	
 	
 	private void writeObject(ObjectOutputStream oos) throws IOException { // ?
@@ -114,38 +117,121 @@ public class OrthoMapsCollection  implements Serializable{
 	}
 	
 	public OrthoMapsCollection(
-			String path) {
+			String path, // lit file
+			OrthoMapsCollection orthoMapsCollection) { // if null will create new, if not null - will update other  
 		if (path.endsWith(".list")) {
-			String[] scenes0 = new String[1];
+//			String[] scenes0 = new String[1];
 			kernels = new ArrayList<AltitudeMismatchKernel>(); 
 			patterns = new ArrayList<GroundObjectPattern>();
-			
+			model_regexes = new HashMap<String,ModelRegex>();
+			String [] regex_use_p = new String[1];
 			String[] paths = getPathsAndScenesFromSourceList(
 					path,
-					scenes0,
+					null, // scenes0,
 					kernels,       // ArrayList<AltitudeMismatchKernel> kernels,
-					patterns); // ArrayList<GroundObjectPattern>    patterns);
-			
-			String scenes_path = scenes0[0];
-			double [][] affine = {{1,0,0},{0,1,0}}; // maybe later calculate from mage_enuatr
+					patterns,      // ArrayList<GroundObjectPattern>    patterns);
+					model_regexes, // ArrayList<ModelRegex>             model_regexes,
+					regex_use_p); // String []                         regex_use) {
+			current_model_regex = regex_use_p[0];
+			paths = ModelRegex.removeDuplicatesSort(paths);        
+			String [] orig_paths = paths.clone();
+			boolean [] replaced = new boolean [paths.length];
+			if (current_model_regex != null) {
+				ModelRegex regex = model_regexes.get(current_model_regex);
+				int num_replaced = 0;
+				int num_changed = 0;
+				if (regex != null) {
+					for (int i = 0; i < paths.length;i++) {
+						String replacement = regex.replaceByFilter(paths[i]);
+						if (replacement != null) {
+							paths[i] = replacement; 
+							replaced[i] = true;
+							num_replaced++;
+							if (!replacement.equals(orig_paths[i])) {
+								num_changed++;
+							}
+						}
+					}
+					System.out.println(String.format("Replacing with regex \"%s\" (%s)",
+							regex.name, regex.regex));
+					System.out.println(String.format("Replaced %d files (of %d), changed %d.", num_replaced, paths.length,num_changed));
+				} else {
+					System.out.println("Replacement regex \""+current_model_regex+"\" does not exist");
+				}
+			}
+//			String scenes_path = scenes0[0]; // not used!
+			ArrayList<String> new_paths = new ArrayList<String>();
+			if (orthoMapsCollection != null) {
+				 // 1. update existing maps
+				for (int i = 0; i < paths.length; i++) {
+					String name = OrthoMap.getNameFromPath(paths[i]);
+					int indx = orthoMapsCollection.getIndex(name);
+					if (indx >= 0) {
+						OrthoMap omap = orthoMapsCollection.ortho_maps[indx];
+						omap.orig_image = new FloatImageData(path); // always replace (nothing to preserve
+						omap.dt = omap.orig_image.getDT(); // just in case?
+						// see if alt image is already initialized
+						if (omap.alt_image != null) {
+							FloatImageData alt_image = OrthoMap.getAlt(path);
+							if (alt_image != null) {
+								omap.alt_image = alt_image;
+							} else {
+								System.out.println("New altitude file does not exist, keeping old one: "+omap.alt_image.path);
+							}
+						}
+					} else {
+						new_paths.add(paths[i]);
+					}
+				}
+			 // 2. add new maps (none will be removed)
+				if (!new_paths.isEmpty()) {
+					// need to create new maps, increase orthoMapsCollection.ortho_maps, reorder and reindex it
+					int num_old_maps = orthoMapsCollection.ortho_maps.length;
+					int num_new_maps = new_paths.size();
+					ortho_maps = new OrthoMap[num_old_maps+num_new_maps];
+					System.arraycopy(
+							orthoMapsCollection.ortho_maps, 0,
+							ortho_maps, 0,
+							num_old_maps);
+					int map_index = num_old_maps;
+					for (String scene_path: new_paths) {
+						ortho_maps[map_index] = new OrthoMap(scene_path); // , scene_path);
+						ortho_maps[map_index++].setAffine();
+					}
+					Arrays.sort(ortho_maps); // mixed old+new, previous scene numbers will no longer be valui
+					orthoMapsCollection.ortho_maps = ortho_maps; 
+					orthoMapsCollection.reindex();
+				} else {
+					System.out.println("No new scenes added");
+				}
+				return; // Constructed this should not be used, only the old orthoMapsCollection
+			}
 			ortho_maps = new OrthoMap[paths.length];
-			
 			for (int n = 0; n < ortho_maps.length; n++) {
-				String name = OrthoMap.getNameFromPath(paths[n]);
 				ortho_maps[n] = new OrthoMap(paths[n]); // , scene_path);
-				ortho_maps[n].setAffine(affine);
+				ortho_maps[n].setAffine();
 			}
+			Arrays.sort(ortho_maps);
+			reindex();
 		} else {
 			System.out.println("OrthoMapsCollection(): path should end with \".list\"");
 		}
-		Arrays.sort(ortho_maps);
-		reindex();
 	}
 	
+	
 	public static double [][] unityAffine() {
 		return new double [][] {{1,0,0},{0,1,0}};
 	}
 	
+	public void updateFiles (
+			String path) {
+		if (path.endsWith(".list")) {
+			new OrthoMapsCollection(path, this); // will update this instead of creating a new instance
+		} else {
+			System.out.println("updateFiles(): path should end with \".list\"");
+		}
+	}
+	
 	// add update kernels, update patterns
 	public void updateKernels (
 			String path) {
@@ -155,7 +241,9 @@ public class OrthoMapsCollection  implements Serializable{
 					path,
 					null,
 					kernels, // ArrayList<AltitudeMismatchKernel> kernels,
-					null);   // ArrayList<GroundObjectPattern>    patterns);
+					null,   // ArrayList<GroundObjectPattern>    patterns);
+					null,    // ArrayList<ModelRegex>             model_regexes,
+					null);   // String []                         regex_use) {
 		} else {
 			System.out.println("updateKernels(): path should end with \".list\"");
 		}
@@ -169,7 +257,10 @@ public class OrthoMapsCollection  implements Serializable{
 					path,
 					null,
 					null, // ArrayList<AltitudeMismatchKernel> kernels,
-					patterns);   // ArrayList<GroundObjectPattern>    patterns);
+					patterns,  // ArrayList<GroundObjectPattern>    patterns);
+					null,    // ArrayList<ModelRegex>             model_regexes,
+					null);   // String []                         regex_use) {
+					
 		} else {
 			System.out.println("updatePatterns(): path should end with \".list\"");
 		}
@@ -269,7 +360,9 @@ public class OrthoMapsCollection  implements Serializable{
 			String files_list,
 			String[] scenes0,
 			ArrayList<AltitudeMismatchKernel> kernels,
-			ArrayList<GroundObjectPattern>    patterns) {
+			ArrayList<GroundObjectPattern>    patterns,
+			HashMap<String,ModelRegex>        model_regexes,
+			String []                         regex_use) {
 		List<String> lines;
 		List<String> rel_files = new ArrayList<String>(); 
 		Path seq_path = Paths.get(files_list);
@@ -311,6 +404,19 @@ public class OrthoMapsCollection  implements Serializable{
 								tokens[3],
 								LocalDateTime.parse(tokens[4])));
 					}
+					
+				} else if ((tokens.length > 0) && (tokens[0].toUpperCase().equals("REGEX"))) {
+					if ((model_regexes != null) && (tokens.length > 4)) {
+						model_regexes.put(tokens[1],new  ModelRegex(
+								tokens[1],                        // String        name,
+								tokens[2],                        //	String        regex,
+								LocalDateTime.parse(tokens[3]),   // LocalDateTime start_date,
+								LocalDateTime.parse(tokens[4]))); // 	LocalDateTime end_date);
+					}
+				} else if ((tokens.length > 0) && (tokens[0].toUpperCase().equals("REGEX_USE"))) {
+					if ((regex_use != null) && (tokens.length > 1)) {
+						regex_use[0] = tokens[1];
+					}					
 				} else if ((tokens.length == 1) && (tokens[0].length() > 0)) {
 					rel_files.add(tokens[0]);
 				}