continue refactoring, migrating to more flexible ortho code (f7200d40) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/orthomosaic/ComboMatch.java

+10 −5

Original line number	Diff line number	Diff line
		@@ -53,6 +53,7 @@ public class ComboMatch {
		public static int gpu_max_width= 4096;
		public static int gpu_max_height= 4096;
		public static String [] FILES_LISTS_PATHS = {
		"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/nov3_50-75-orange",
		"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/nov3_50-75",
		"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/maps_sep12-13_50-25-50-75-100m",
		"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/maps_19_sep13_25-50-75-100m",
		@@ -176,8 +177,8 @@ public class ComboMatch {



		boolean use_saved_collection = true; // false;
		boolean save_collection = true;
		boolean use_saved_collection = false; // true; // false;
		boolean save_collection = false; // true;
		boolean process_correlation = true; // use false to save new version of data
		int num_tries_fit = 10;
		boolean update_match = true; // use false to save new version of data
		@@ -190,6 +191,7 @@ public class ComboMatch {
		boolean restore_temp = true;
		double frac_remove = clt_parameters.imp.pmtch_frac_remove; // 0.15;
		double metric_error = clt_parameters.imp.pmtch_metric_err; // 0.05; // 0.02;// 2 cm
		boolean update_files = false;
		boolean update_lla = false; // re-read file metadata
		boolean update_kernel_patterns = false;
		boolean update_bl_bc = false;
		@@ -248,6 +250,7 @@ public class ComboMatch {
		if (use_marked_image ) {
		gd.addCheckbox ("Use marked image data", true, "Use markes from the selected image");
		}
		gd.addCheckbox ("Update scene files", update_files, "Re-read files as specified in .list file, possibly changing versions");
		gd.addCheckbox ("Update files metadata", update_lla, "Re-read files metadata (if it was modified)");
		gd.addCheckbox ("Update kernels/patterns", update_kernel_patterns, "Re-read kernels and patterns from *.list file");
		gd.addCheckbox ("Remove duplicate scenes", fix_duplicates, "Remove scenes with the same timestamp");
		@@ -304,6 +307,7 @@ public class ComboMatch {
		if (use_marked_image ) { // will only be used if found and asked
		use_marked_image= gd.getNextBoolean();
		}
		update_files= gd.getNextBoolean();
		update_lla= gd.getNextBoolean();
		update_kernel_patterns= gd.getNextBoolean();
		fix_duplicates= gd.getNextBoolean();
		@@ -321,8 +325,6 @@ public class ComboMatch {
		} catch (ClassNotFoundException \| IOException e) {
		use_files_list = true;
		System.out.println("Saved data not found: "+orthoMapsCollection_path+", building from a list "+files_list_path);
		// TODO Auto-generated catch block
		// e.printStackTrace();
		}
		//files_list_path
		if (!use_files_list) {
		@@ -330,12 +332,15 @@ public class ComboMatch {
		maps_collection.updateKernels(files_list_path);
		maps_collection.updatePatterns(files_list_path);
		}
		if (update_files) {
		maps_collection.updateFiles(files_list_path);
		}
		}
		} else {
		use_files_list = true;
		}
		if (use_files_list) { // using files list
		maps_collection = new OrthoMapsCollection(files_list_path); // should have ".list" extension
		maps_collection = new OrthoMapsCollection(files_list_path, null); // should have ".list" extension
		maps_collection.updateNumberScenes();
		maps_collection.updateSfmGain();
		}

src/main/java/com/elphel/imagej/orthomosaic/ModelRegex.java

0 → 100644

+126 −0

Original line number	Diff line number	Diff line
		package com.elphel.imagej.orthomosaic;

		import java.io.File;
		import java.io.IOException;
		import java.io.ObjectInputStream;
		import java.io.ObjectOutputStream;
		import java.io.Serializable;
		import java.nio.file.attribute.FileTime;
		import java.text.SimpleDateFormat;
		import java.time.Instant;
		import java.time.LocalDateTime;
		import java.time.ZoneId;
		import java.time.format.DateTimeFormatter;
		import java.util.Arrays;
		import java.util.Calendar;
		import java.util.Collection;
		import java.util.Date;
		import java.util.HashMap;
		import java.util.regex.Pattern;

		import ij.Prefs;

		public class ModelRegex implements Serializable{
		private static final long serialVersionUID = 1L;
		public String name;
		public String regex;
		public LocalDateTime start_date;
		public LocalDateTime end_date;

		public ModelRegex(
		String name,
		String regex,
		LocalDateTime start_date,
		LocalDateTime end_date) {
		this.name = name;
		this.regex = regex;
		this.start_date = start_date;
		this.end_date = end_date;
		}

		private void writeObject(ObjectOutputStream oos) throws IOException {
		oos.defaultWriteObject();
		}
		private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
		ois.defaultReadObject();
		}

		public String replaceByFilter (String path) {
		String model_path = OrthoMap.getModelPathFromPath(path);
		File model_dir = (new File(model_path)); // .getParentFile();
		if (!model_dir.exists() \|\| !model_dir.isDirectory()) {
		return null;
		}
		File [] files = model_dir.listFiles();
		boolean [] matches = new boolean [files.length];
		int num_matches = 0;
		Pattern filename_pattern= Pattern.compile(regex);
		int latest=-1;
		LocalDateTime [] dts = new LocalDateTime[files.length];
		for (int i = 0; i < files.length; i++) if (files[i].isFile()){
		long fileLastModifiedDate = files[i].lastModified();
		dts[i] = Instant.ofEpochMilli(fileLastModifiedDate).atZone(ZoneId.systemDefault()).toLocalDateTime();
		if ((start_date != null) && dts[i].isBefore(start_date)){
		continue; // too early
		}
		if ((end_date != null) && dts[i].isAfter(end_date)){
		continue; // too late
		}
		if (!filename_pattern.matcher(files[i].getName()).matches()) {
		continue; // does not match
		}
		matches[i] = true;
		num_matches++;
		if ((latest < 0) \|\| dts[i].isAfter(dts[latest])) {
		latest = i;
		}
		}
		if (num_matches >1) {
		System.out.println("dir="+model_dir+", num_matches="+num_matches);
		}
		if (latest >= 0) {
		//Prefs.getFileSeparator()
		return model_path+Prefs.getFileSeparator()+files[latest].getName();
		}
		// find latest
		return null;
		}

		public static String [] removeDuplicatesSort(String [] paths) {
		HashMap<String,String> scene_files = new HashMap<String,String>();
		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss.SS zzz");// may be VV instead of zzz
		for (int i = 0; i < paths.length; i++) {

		String sts = OrthoMap.getNameFromPath(paths[i]);
		String model_dir = OrthoMap.getModelPathFromPath(paths[i]);
		if (scene_files.containsKey(sts)) {
		String path_old = scene_files.get(sts);
		long old_date = (new File(path_old)).lastModified();
		long new_date = (new File(paths[i])).lastModified();
		boolean replace = new_date > old_date;
		LocalDateTime old_dt=Instant.ofEpochMilli(old_date).atZone(ZoneId.systemDefault()).toLocalDateTime();
		LocalDateTime new_dt=Instant.ofEpochMilli(new_date).atZone(ZoneId.systemDefault()).toLocalDateTime();
		System.out.println("Duplicate scene found: "+sts+" in two files:");
		System.out.println("1: "+path_old+" @ "+old_dt.atZone(ZoneId.systemDefault()).format(formatter));
		System.out.println("2: "+paths[i]+" @ "+new_dt.atZone(ZoneId.systemDefault()).format(formatter));
		System.out.println("Using "+(replace ? 2 : 1) +" as it is not older");
		if (replace) {
		scene_files.put(sts, paths[i]);
		}
		} else {
		scene_files.put(sts, paths[i]);
		}
		}
		// assuming string ts have the same order as double timestamps
		String [] filtered_ts = scene_files.keySet().toArray(new String[0]);
		Arrays.sort(filtered_ts);
		String [] filtered_paths = new String [filtered_ts.length];
		for (int i = 0; i < filtered_paths.length; i++) {
		filtered_paths[i] = scene_files.get(filtered_ts[i]);
		}
		return filtered_paths;
		}



		}

src/main/java/com/elphel/imagej/orthomosaic/OrthoMap.java

+18 −8

Original line number	Diff line number	Diff line
		@@ -588,17 +588,22 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
		*/
		public FloatImageData getAlt() {
		if (alt_image == null) {
		String alt_path = getAltPath(path);
		alt_image = getAlt(path); // img path, not alt
		}
		return alt_image;
		}

		public static FloatImageData getAlt(String img_path) {
		String alt_path = getAltPath(img_path);
		File alt_file = new File(alt_path);
		if (!alt_file.exists()) {
		System.out.println("Altitudes file does not exist: "+alt_path);
		return null;
		}
		alt_image = new FloatImageData(alt_path);
		}
		return orig_image;
		return new FloatImageData(alt_path);
		}


		/*
		public OrthoMap (
		String path) {
		@@ -676,6 +681,11 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
		public void setAffine(double [][] affine) {
		this.affine = new double[][] {affine[0].clone(),affine[1].clone()};
		}

		public void setAffine() {
		this.affine = new double[][] {{1,0,0},{0,1,0}};
		}

		public double [][] getAffine(){
		return affine;
		}

src/main/java/com/elphel/imagej/orthomosaic/OrthoMapsCollection.java

+123 −17

Original line number	Diff line number	Diff line
		@@ -86,10 +86,13 @@ public class OrthoMapsCollection implements Serializable{
		public static final String [] NAME_MO = {"main","other"};
		public transient long version = -1;
		OrthoMap [] ortho_maps;
		// remove some transients
		transient HashMap<Double,Integer> map_index;
		transient HashMap<String,Integer> map_index_string;
		transient ArrayList<AltitudeMismatchKernel> kernels;
		transient ArrayList<GroundObjectPattern> patterns;
		transient HashMap<String,ModelRegex> model_regexes;
		transient String current_model_regex="";


		private void writeObject(ObjectOutputStream oos) throws IOException { // ?
		@@ -114,38 +117,121 @@ public class OrthoMapsCollection implements Serializable{
		}

		public OrthoMapsCollection(
		String path) {
		String path, // lit file
		OrthoMapsCollection orthoMapsCollection) { // if null will create new, if not null - will update other
		if (path.endsWith(".list")) {
		String[] scenes0 = new String[1];
		// String[] scenes0 = new String[1];
		kernels = new ArrayList<AltitudeMismatchKernel>();
		patterns = new ArrayList<GroundObjectPattern>();

		model_regexes = new HashMap<String,ModelRegex>();
		String [] regex_use_p = new String[1];
		String[] paths = getPathsAndScenesFromSourceList(
		path,
		scenes0,
		null, // scenes0,
		kernels, // ArrayList<AltitudeMismatchKernel> kernels,
		patterns); // ArrayList<GroundObjectPattern> patterns);

		String scenes_path = scenes0[0];
		double [][] affine = {{1,0,0},{0,1,0}}; // maybe later calculate from mage_enuatr
		patterns, // ArrayList<GroundObjectPattern> patterns);
		model_regexes, // ArrayList<ModelRegex> model_regexes,
		regex_use_p); // String [] regex_use) {
		current_model_regex = regex_use_p[0];
		paths = ModelRegex.removeDuplicatesSort(paths);
		String [] orig_paths = paths.clone();
		boolean [] replaced = new boolean [paths.length];
		if (current_model_regex != null) {
		ModelRegex regex = model_regexes.get(current_model_regex);
		int num_replaced = 0;
		int num_changed = 0;
		if (regex != null) {
		for (int i = 0; i < paths.length;i++) {
		String replacement = regex.replaceByFilter(paths[i]);
		if (replacement != null) {
		paths[i] = replacement;
		replaced[i] = true;
		num_replaced++;
		if (!replacement.equals(orig_paths[i])) {
		num_changed++;
		}
		}
		}
		System.out.println(String.format("Replacing with regex \"%s\" (%s)",
		regex.name, regex.regex));
		System.out.println(String.format("Replaced %d files (of %d), changed %d.", num_replaced, paths.length,num_changed));
		} else {
		System.out.println("Replacement regex \""+current_model_regex+"\" does not exist");
		}
		}
		// String scenes_path = scenes0[0]; // not used!
		ArrayList<String> new_paths = new ArrayList<String>();
		if (orthoMapsCollection != null) {
		// 1. update existing maps
		for (int i = 0; i < paths.length; i++) {
		String name = OrthoMap.getNameFromPath(paths[i]);
		int indx = orthoMapsCollection.getIndex(name);
		if (indx >= 0) {
		OrthoMap omap = orthoMapsCollection.ortho_maps[indx];
		omap.orig_image = new FloatImageData(path); // always replace (nothing to preserve
		omap.dt = omap.orig_image.getDT(); // just in case?
		// see if alt image is already initialized
		if (omap.alt_image != null) {
		FloatImageData alt_image = OrthoMap.getAlt(path);
		if (alt_image != null) {
		omap.alt_image = alt_image;
		} else {
		System.out.println("New altitude file does not exist, keeping old one: "+omap.alt_image.path);
		}
		}
		} else {
		new_paths.add(paths[i]);
		}
		}
		// 2. add new maps (none will be removed)
		if (!new_paths.isEmpty()) {
		// need to create new maps, increase orthoMapsCollection.ortho_maps, reorder and reindex it
		int num_old_maps = orthoMapsCollection.ortho_maps.length;
		int num_new_maps = new_paths.size();
		ortho_maps = new OrthoMap[num_old_maps+num_new_maps];
		System.arraycopy(
		orthoMapsCollection.ortho_maps, 0,
		ortho_maps, 0,
		num_old_maps);
		int map_index = num_old_maps;
		for (String scene_path: new_paths) {
		ortho_maps[map_index] = new OrthoMap(scene_path); // , scene_path);
		ortho_maps[map_index++].setAffine();
		}
		Arrays.sort(ortho_maps); // mixed old+new, previous scene numbers will no longer be valui
		orthoMapsCollection.ortho_maps = ortho_maps;
		orthoMapsCollection.reindex();
		} else {
		System.out.println("No new scenes added");
		}
		return; // Constructed this should not be used, only the old orthoMapsCollection
		}
		ortho_maps = new OrthoMap[paths.length];

		for (int n = 0; n < ortho_maps.length; n++) {
		String name = OrthoMap.getNameFromPath(paths[n]);
		ortho_maps[n] = new OrthoMap(paths[n]); // , scene_path);
		ortho_maps[n].setAffine(affine);
		ortho_maps[n].setAffine();
		}
		Arrays.sort(ortho_maps);
		reindex();
		} else {
		System.out.println("OrthoMapsCollection(): path should end with \".list\"");
		}
		Arrays.sort(ortho_maps);
		reindex();
		}


		public static double [][] unityAffine() {
		return new double [][] {{1,0,0},{0,1,0}};
		}

		public void updateFiles (
		String path) {
		if (path.endsWith(".list")) {
		new OrthoMapsCollection(path, this); // will update this instead of creating a new instance
		} else {
		System.out.println("updateFiles(): path should end with \".list\"");
		}
		}

		// add update kernels, update patterns
		public void updateKernels (
		String path) {
		@@ -155,7 +241,9 @@ public class OrthoMapsCollection implements Serializable{
		path,
		null,
		kernels, // ArrayList<AltitudeMismatchKernel> kernels,
		null); // ArrayList<GroundObjectPattern> patterns);
		null, // ArrayList<GroundObjectPattern> patterns);
		null, // ArrayList<ModelRegex> model_regexes,
		null); // String [] regex_use) {
		} else {
		System.out.println("updateKernels(): path should end with \".list\"");
		}
		@@ -169,7 +257,10 @@ public class OrthoMapsCollection implements Serializable{
		path,
		null,
		null, // ArrayList<AltitudeMismatchKernel> kernels,
		patterns); // ArrayList<GroundObjectPattern> patterns);
		patterns, // ArrayList<GroundObjectPattern> patterns);
		null, // ArrayList<ModelRegex> model_regexes,
		null); // String [] regex_use) {

		} else {
		System.out.println("updatePatterns(): path should end with \".list\"");
		}
		@@ -269,7 +360,9 @@ public class OrthoMapsCollection implements Serializable{
		String files_list,
		String[] scenes0,
		ArrayList<AltitudeMismatchKernel> kernels,
		ArrayList<GroundObjectPattern> patterns) {
		ArrayList<GroundObjectPattern> patterns,
		HashMap<String,ModelRegex> model_regexes,
		String [] regex_use) {
		List<String> lines;
		List<String> rel_files = new ArrayList<String>();
		Path seq_path = Paths.get(files_list);
		@@ -311,6 +404,19 @@ public class OrthoMapsCollection implements Serializable{
		tokens[3],
		LocalDateTime.parse(tokens[4])));
		}

		} else if ((tokens.length > 0) && (tokens[0].toUpperCase().equals("REGEX"))) {
		if ((model_regexes != null) && (tokens.length > 4)) {
		model_regexes.put(tokens[1],new ModelRegex(
		tokens[1], // String name,
		tokens[2], // String regex,
		LocalDateTime.parse(tokens[3]), // LocalDateTime start_date,
		LocalDateTime.parse(tokens[4]))); // LocalDateTime end_date);
		}
		} else if ((tokens.length > 0) && (tokens[0].toUpperCase().equals("REGEX_USE"))) {
		if ((regex_use != null) && (tokens.length > 1)) {
		regex_use[0] = tokens[1];
		}
		} else if ((tokens.length == 1) && (tokens[0].length() > 0)) {
		rel_files.add(tokens[0]);
		}

Admin message