Skip to content

I
imagej-elphel
Commit f7200d40 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

continue refactoring, migrating to more flexible ortho code

parent c5431a27
Hide whitespace changes
Inline Side-by-side
Showing with 277 additions and 30 deletions
src/main/java/com/elphel/imagej/orthomosaic/ComboMatch.java
View file @ f7200d40
......@@ -53,6 +53,7 @@ public class ComboMatch {
public static int gpu_max_width= 4096;
public static int gpu_max_height= 4096;
public static String [] FILES_LISTS_PATHS = {
"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/nov3_50-75-orange",
"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/nov3_50-75",
"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/maps_sep12-13_50-25-50-75-100m",
"/media/elphel/SSD3-4GB/lwir16-proc/ortho_videos/maps_19_sep13_25-50-75-100m",
......@@ -176,8 +177,8 @@ public class ComboMatch {
boolean use_saved_collection = true; // false;
boolean save_collection = true;
boolean use_saved_collection = false; // true; // false;
boolean save_collection = false; // true;
boolean process_correlation = true; // use false to save new version of data
int num_tries_fit = 10;
boolean update_match = true; // use false to save new version of data
......@@ -190,6 +191,7 @@ public class ComboMatch {
boolean restore_temp = true;
double frac_remove = clt_parameters.imp.pmtch_frac_remove; // 0.15;
double metric_error = clt_parameters.imp.pmtch_metric_err; // 0.05; // 0.02;// 2 cm
boolean update_files = false;
boolean update_lla = false; // re-read file metadata
boolean update_kernel_patterns = false;
boolean update_bl_bc = false;
......@@ -248,6 +250,7 @@ public class ComboMatch {
if (use_marked_image ) {
gd.addCheckbox ("Use marked image data", true, "Use markes from the selected image");
}
gd.addCheckbox ("Update scene files", update_files, "Re-read files as specified in .list file, possibly changing versions");
gd.addCheckbox ("Update files metadata", update_lla, "Re-read files metadata (if it was modified)");
gd.addCheckbox ("Update kernels/patterns", update_kernel_patterns, "Re-read kernels and patterns from *.list file");
gd.addCheckbox ("Remove duplicate scenes", fix_duplicates, "Remove scenes with the same timestamp");
......@@ -304,6 +307,7 @@ public class ComboMatch {
if (use_marked_image ) { // will only be used if found and asked
use_marked_image= gd.getNextBoolean();
}
update_files= gd.getNextBoolean();
update_lla= gd.getNextBoolean();
update_kernel_patterns= gd.getNextBoolean();
fix_duplicates= gd.getNextBoolean();
......@@ -321,8 +325,6 @@ public class ComboMatch {
} catch (ClassNotFoundException | IOException e) {
use_files_list = true;
System.out.println("Saved data not found: "+orthoMapsCollection_path+", building from a list "+files_list_path);
// TODO Auto-generated catch block
// e.printStackTrace();
}
//files_list_path
if (!use_files_list) {
......@@ -330,12 +332,15 @@ public class ComboMatch {
maps_collection.updateKernels(files_list_path);
maps_collection.updatePatterns(files_list_path);
}
if (update_files) {
maps_collection.updateFiles(files_list_path);
}
}
} else {
use_files_list = true;
}
if (use_files_list) { // using files list
maps_collection = new OrthoMapsCollection(files_list_path); // should have ".list" extension
maps_collection = new OrthoMapsCollection(files_list_path, null); // should have ".list" extension
maps_collection.updateNumberScenes();
maps_collection.updateSfmGain();
}
......
src/main/java/com/elphel/imagej/orthomosaic/ModelRegex.java 0 → 100644
View file @ f7200d40
package com.elphel.imagej.orthomosaic;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.file.attribute.FileTime;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.regex.Pattern;
import ij.Prefs;
public class ModelRegex implements Serializable{
private static final long serialVersionUID = 1L;
public String name;
public String regex;
public LocalDateTime start_date;
public LocalDateTime end_date;
public ModelRegex(
String name,
String regex,
LocalDateTime start_date,
LocalDateTime end_date) {
this.name = name;
this.regex = regex;
this.start_date = start_date;
this.end_date = end_date;
}
private void writeObject(ObjectOutputStream oos) throws IOException {
oos.defaultWriteObject();
}
private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
ois.defaultReadObject();
}
public String replaceByFilter (String path) {
String model_path = OrthoMap.getModelPathFromPath(path);
File model_dir = (new File(model_path)); // .getParentFile();
if (!model_dir.exists() || !model_dir.isDirectory()) {
return null;
}
File [] files = model_dir.listFiles();
boolean [] matches = new boolean [files.length];
int num_matches = 0;
Pattern filename_pattern= Pattern.compile(regex);
int latest=-1;
LocalDateTime [] dts = new LocalDateTime[files.length];
for (int i = 0; i < files.length; i++) if (files[i].isFile()){
long fileLastModifiedDate = files[i].lastModified();
dts[i] = Instant.ofEpochMilli(fileLastModifiedDate).atZone(ZoneId.systemDefault()).toLocalDateTime();
if ((start_date != null) && dts[i].isBefore(start_date)){
continue; // too early
}
if ((end_date != null) && dts[i].isAfter(end_date)){
continue; // too late
}
if (!filename_pattern.matcher(files[i].getName()).matches()) {
continue; // does not match
}
matches[i] = true;
num_matches++;
if ((latest < 0) || dts[i].isAfter(dts[latest])) {
latest = i;
}
}
if (num_matches >1) {
System.out.println("dir="+model_dir+", num_matches="+num_matches);
}
if (latest >= 0) {
//Prefs.getFileSeparator()
return model_path+Prefs.getFileSeparator()+files[latest].getName();
}
// find latest
return null;
}
public static String [] removeDuplicatesSort(String [] paths) {
HashMap<String,String> scene_files = new HashMap<String,String>();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss.SS zzz");// may be VV instead of zzz
for (int i = 0; i < paths.length; i++) {
String sts = OrthoMap.getNameFromPath(paths[i]);
String model_dir = OrthoMap.getModelPathFromPath(paths[i]);
if (scene_files.containsKey(sts)) {
String path_old = scene_files.get(sts);
long old_date = (new File(path_old)).lastModified();
long new_date = (new File(paths[i])).lastModified();
boolean replace = new_date > old_date;
LocalDateTime old_dt=Instant.ofEpochMilli(old_date).atZone(ZoneId.systemDefault()).toLocalDateTime();
LocalDateTime new_dt=Instant.ofEpochMilli(new_date).atZone(ZoneId.systemDefault()).toLocalDateTime();
System.out.println("Duplicate scene found: "+sts+" in two files:");
System.out.println("1: "+path_old+" @ "+old_dt.atZone(ZoneId.systemDefault()).format(formatter));
System.out.println("2: "+paths[i]+" @ "+new_dt.atZone(ZoneId.systemDefault()).format(formatter));
System.out.println("Using "+(replace ? 2 : 1) +" as it is not older");
if (replace) {
scene_files.put(sts, paths[i]);
}
} else {
scene_files.put(sts, paths[i]);
}
}
// assuming string ts have the same order as double timestamps
String [] filtered_ts = scene_files.keySet().toArray(new String[0]);
Arrays.sort(filtered_ts);
String [] filtered_paths = new String [filtered_ts.length];
for (int i = 0; i < filtered_paths.length; i++) {
filtered_paths[i] = scene_files.get(filtered_ts[i]);
}
return filtered_paths;
}
}
src/main/java/com/elphel/imagej/orthomosaic/OrthoMap.java
View file @ f7200d40
......@@ -588,17 +588,22 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
*/
public FloatImageData getAlt() {
if (alt_image == null) {
String alt_path = getAltPath(path);
File alt_file = new File(alt_path);
if (!alt_file.exists()) {
System.out.println("Altitudes file does not exist: "+alt_path);
return null;
}
alt_image = new FloatImageData(alt_path);
alt_image = getAlt(path); // img path, not alt
}
return orig_image;
return alt_image;
}
public static FloatImageData getAlt(String img_path) {
String alt_path = getAltPath(img_path);
File alt_file = new File(alt_path);
if (!alt_file.exists()) {
System.out.println("Altitudes file does not exist: "+alt_path);
return null;
}
return new FloatImageData(alt_path);
}
/*
public OrthoMap (
String path) {
......@@ -676,6 +681,11 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
public void setAffine(double [][] affine) {
this.affine = new double[][] {affine[0].clone(),affine[1].clone()};
}
public void setAffine() {
this.affine = new double[][] {{1,0,0},{0,1,0}};
}
public double [][] getAffine(){
return affine;
}
......
src/main/java/com/elphel/imagej/orthomosaic/OrthoMapsCollection.java
View file @ f7200d40
......@@ -86,10 +86,13 @@ public class OrthoMapsCollection implements Serializable{
public static final String [] NAME_MO = {"main","other"};
public transient long version = -1;
OrthoMap [] ortho_maps;
// remove some transients
transient HashMap<Double,Integer> map_index;
transient HashMap<String,Integer> map_index_string;
transient ArrayList<AltitudeMismatchKernel> kernels;
transient ArrayList<GroundObjectPattern> patterns;
transient HashMap<String,ModelRegex> model_regexes;
transient String current_model_regex="";
private void writeObject(ObjectOutputStream oos) throws IOException { // ?
......@@ -114,38 +117,121 @@ public class OrthoMapsCollection implements Serializable{
}
public OrthoMapsCollection(
String path) {
String path, // lit file
OrthoMapsCollection orthoMapsCollection) { // if null will create new, if not null - will update other
if (path.endsWith(".list")) {
String[] scenes0 = new String[1];
// String[] scenes0 = new String[1];
kernels = new ArrayList<AltitudeMismatchKernel>();
patterns = new ArrayList<GroundObjectPattern>();
model_regexes = new HashMap<String,ModelRegex>();
String [] regex_use_p = new String[1];
String[] paths = getPathsAndScenesFromSourceList(
path,
scenes0,
kernels, // ArrayList<AltitudeMismatchKernel> kernels,
patterns); // ArrayList<GroundObjectPattern> patterns);
String scenes_path = scenes0[0];
double [][] affine = {{1,0,0},{0,1,0}}; // maybe later calculate from mage_enuatr
null, // scenes0,
kernels, // ArrayList<AltitudeMismatchKernel> kernels,
patterns, // ArrayList<GroundObjectPattern> patterns);
model_regexes, // ArrayList<ModelRegex> model_regexes,
regex_use_p); // String [] regex_use) {
current_model_regex = regex_use_p[0];
paths = ModelRegex.removeDuplicatesSort(paths);
String [] orig_paths = paths.clone();
boolean [] replaced = new boolean [paths.length];
if (current_model_regex != null) {
ModelRegex regex = model_regexes.get(current_model_regex);
int num_replaced = 0;
int num_changed = 0;
if (regex != null) {
for (int i = 0; i < paths.length;i++) {
String replacement = regex.replaceByFilter(paths[i]);
if (replacement != null) {
paths[i] = replacement;
replaced[i] = true;
num_replaced++;
if (!replacement.equals(orig_paths[i])) {
num_changed++;
}
}
}
System.out.println(String.format("Replacing with regex \"%s\" (%s)",
regex.name, regex.regex));
System.out.println(String.format("Replaced %d files (of %d), changed %d.", num_replaced, paths.length,num_changed));
} else {
System.out.println("Replacement regex \""+current_model_regex+"\" does not exist");
}
}
// String scenes_path = scenes0[0]; // not used!
ArrayList<String> new_paths = new ArrayList<String>();
if (orthoMapsCollection != null) {
// 1. update existing maps
for (int i = 0; i < paths.length; i++) {
String name = OrthoMap.getNameFromPath(paths[i]);
int indx = orthoMapsCollection.getIndex(name);
if (indx >= 0) {
OrthoMap omap = orthoMapsCollection.ortho_maps[indx];
omap.orig_image = new FloatImageData(path); // always replace (nothing to preserve
omap.dt = omap.orig_image.getDT(); // just in case?
// see if alt image is already initialized
if (omap.alt_image != null) {
FloatImageData alt_image = OrthoMap.getAlt(path);
if (alt_image != null) {
omap.alt_image = alt_image;
} else {
System.out.println("New altitude file does not exist, keeping old one: "+omap.alt_image.path);
}
}
} else {
new_paths.add(paths[i]);
}
}
// 2. add new maps (none will be removed)
if (!new_paths.isEmpty()) {
// need to create new maps, increase orthoMapsCollection.ortho_maps, reorder and reindex it
int num_old_maps = orthoMapsCollection.ortho_maps.length;
int num_new_maps = new_paths.size();
ortho_maps = new OrthoMap[num_old_maps+num_new_maps];
System.arraycopy(
orthoMapsCollection.ortho_maps, 0,
ortho_maps, 0,
num_old_maps);
int map_index = num_old_maps;
for (String scene_path: new_paths) {
ortho_maps[map_index] = new OrthoMap(scene_path); // , scene_path);
ortho_maps[map_index++].setAffine();
}
Arrays.sort(ortho_maps); // mixed old+new, previous scene numbers will no longer be valui
orthoMapsCollection.ortho_maps = ortho_maps;
orthoMapsCollection.reindex();
} else {
System.out.println("No new scenes added");
}
return; // Constructed this should not be used, only the old orthoMapsCollection
}
ortho_maps = new OrthoMap[paths.length];
for (int n = 0; n < ortho_maps.length; n++) {
String name = OrthoMap.getNameFromPath(paths[n]);
ortho_maps[n] = new OrthoMap(paths[n]); // , scene_path);
ortho_maps[n].setAffine(affine);
ortho_maps[n].setAffine();
}
Arrays.sort(ortho_maps);
reindex();
} else {
System.out.println("OrthoMapsCollection(): path should end with \".list\"");
}
Arrays.sort(ortho_maps);
reindex();
}
public static double [][] unityAffine() {
return new double [][] {{1,0,0},{0,1,0}};
}
public void updateFiles (
String path) {
if (path.endsWith(".list")) {
new OrthoMapsCollection(path, this); // will update this instead of creating a new instance
} else {
System.out.println("updateFiles(): path should end with \".list\"");
}
}
// add update kernels, update patterns
public void updateKernels (
String path) {
......@@ -155,7 +241,9 @@ public class OrthoMapsCollection implements Serializable{
path,
null,
kernels, // ArrayList<AltitudeMismatchKernel> kernels,
null); // ArrayList<GroundObjectPattern> patterns);
null, // ArrayList<GroundObjectPattern> patterns);
null, // ArrayList<ModelRegex> model_regexes,
null); // String [] regex_use) {
} else {
System.out.println("updateKernels(): path should end with \".list\"");
}
......@@ -169,7 +257,10 @@ public class OrthoMapsCollection implements Serializable{
path,
null,
null, // ArrayList<AltitudeMismatchKernel> kernels,
patterns); // ArrayList<GroundObjectPattern> patterns);
patterns, // ArrayList<GroundObjectPattern> patterns);
null, // ArrayList<ModelRegex> model_regexes,
null); // String [] regex_use) {
} else {
System.out.println("updatePatterns(): path should end with \".list\"");
}
......@@ -269,7 +360,9 @@ public class OrthoMapsCollection implements Serializable{
String files_list,
String[] scenes0,
ArrayList<AltitudeMismatchKernel> kernels,
ArrayList<GroundObjectPattern> patterns) {
ArrayList<GroundObjectPattern> patterns,
HashMap<String,ModelRegex> model_regexes,
String [] regex_use) {
List<String> lines;
List<String> rel_files = new ArrayList<String>();
Path seq_path = Paths.get(files_list);
......@@ -311,6 +404,19 @@ public class OrthoMapsCollection implements Serializable{
tokens[3],
LocalDateTime.parse(tokens[4])));
}
} else if ((tokens.length > 0) && (tokens[0].toUpperCase().equals("REGEX"))) {
if ((model_regexes != null) && (tokens.length > 4)) {
model_regexes.put(tokens[1],new ModelRegex(
tokens[1], // String name,
tokens[2], // String regex,
LocalDateTime.parse(tokens[3]), // LocalDateTime start_date,
LocalDateTime.parse(tokens[4]))); // LocalDateTime end_date);
}
} else if ((tokens.length > 0) && (tokens[0].toUpperCase().equals("REGEX_USE"))) {
if ((regex_use != null) && (tokens.length > 1)) {
regex_use[0] = tokens[1];
}
} else if ((tokens.length == 1) && (tokens[0].length() > 0)) {
rel_files.add(tokens[0]);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment