Commit e1616ff9 authored by Andrey Filippov's avatar Andrey Filippov

Started with scene sequence overlap

parent f1e78589
......@@ -162,32 +162,77 @@ public class Interscene {
// Index is easier, no need to change
// QuadCLT[] quadCLTs_half = new QuadCLT[(quadCLTs.length+1)/2];
// QuadCLT.SetChannels [] set_channels_half;
// consider if it is using initial DSI from previous in overlap mode
boolean overlap_sequences = clt_parameters.imp.overlap_sequences;
int [] start_ref_pointers1 = new int[2];
// try to read previously adjusted half-range
ErsCorrection ers_reference = quadCLTs[ref_index].getErsCorrection();
int cent_index = earliest_scene + (ref_index - earliest_scene) / 2;
int min_num_scenes_half = min_num_scenes / 2;
// TODO: add ref scene itself to the list?
cent_index = setInitialOrientationsIms(
clt_parameters, // final CLTParameters clt_parameters,
min_num_scenes_half, // int min_num_scenes,
colorProcParameters, // final ColorProcParameters colorProcParameters,
quadCLTs, // final QuadCLT[] quadCLTs, //
ref_index, // final int ref_index,
set_channels, // final QuadCLT.SetChannels [] set_channels,
batch_mode, // final boolean batch_mode,
cent_index, // int earliest_scene,
start_ref_pointers1, // int [] start_ref_pointers, // [0] - earliest valid scene, [1] ref_index
threadsMax, // final int threadsMax, // int threadsMax,
updateStatus, // final boolean updateStatus,
debugLevel); // final int debugLevel)
if (cent_index < 0) {
System.out.println("setInitialOrientationsIms() first half failed. Consider more graceful bail out.");
start_ref_pointers[0] = start_ref_pointers1[0];
return cent_index;
int min_num_scenes_half = min_num_scenes / 2;
boolean reused_overlap = false;
if (overlap_sequences) {
int adjusted_scene_index = ref_index - 1;
for (; adjusted_scene_index >= 0 ; adjusted_scene_index--) {
// to include ref scene photometric calibration
quadCLTs[adjusted_scene_index] = quadCLTs[ref_index].spawnNoModelQuadCLT(
set_channels[adjusted_scene_index].set_name,
clt_parameters,
colorProcParameters, //
threadsMax,
debugLevel-2);
String ts = quadCLTs[adjusted_scene_index].getImageName();
if (ts == null) {
break;
}
if (adjusted_scene_index < (min_num_scenes_half-1)) {
if (debugLevel > -3) {
System.out.println("Overlap mode, too few scenes left for the second half, shortening first to "+
(ref_index - adjusted_scene_index - 1));
}
break;
}
} // split cycles to remove output clutter
adjusted_scene_index++;
if ((ref_index - adjusted_scene_index) >= min_num_scenes_half) {
if (debugLevel > -3) {
System.out.println("Overlap mode, reusing previous scenes adjustment scenes, number of scenes = "+
(ref_index - adjusted_scene_index));
}
cent_index = adjusted_scene_index;
reused_overlap = true;
} else {
if (debugLevel > -3) {
System.out.println("Overlap mode, too few scenes to reuse: "+
(ref_index - adjusted_scene_index)+" < "+ min_num_scenes_half);
}
}
}
if (!reused_overlap) {
// TODO: add ref scene itself to the list?
cent_index = setInitialOrientationsIms(
clt_parameters, // final CLTParameters clt_parameters,
min_num_scenes_half, // int min_num_scenes,
colorProcParameters, // final ColorProcParameters colorProcParameters,
quadCLTs, // final QuadCLT[] quadCLTs, //
ref_index, // final int ref_index,
set_channels, // final QuadCLT.SetChannels [] set_channels,
batch_mode, // final boolean batch_mode,
cent_index, // int earliest_scene,
start_ref_pointers1, // int [] start_ref_pointers, // [0] - earliest valid scene, [1] ref_index
threadsMax, // final int threadsMax, // int threadsMax,
updateStatus, // final boolean updateStatus,
debugLevel); // final int debugLevel)
if (cent_index < 0) {
System.out.println("setInitialOrientationsIms() first half failed. Consider more graceful bail out.");
start_ref_pointers[0] = start_ref_pointers1[0];
return cent_index;
}
}
// create new dsi for quadCLTs[earliest_scene1]
OpticalFlow.buildRefDSI( // returned is a different instance than input -FIXED
clt_parameters, // CLTParameters clt_parameters,
false, // boolean fast,
true, // boolean skip_photo,
debayerParameters, // EyesisCorrectionParameters.DebayerParameters debayerParameters,
colorProcParameters, // ColorProcParameters colorProcParameters,
rgbParameters, // EyesisCorrectionParameters.RGBParameters rgbParameters,
......@@ -237,11 +282,12 @@ public class Interscene {
}
// invert first half, reference to the cent_index, add to cent_index map, generate ref_index ponter and cent_index,
// write config for both ref_index and cent_index scenes
ErsCorrection ers_reference = quadCLTs[ref_index].getErsCorrection();
// ErsCorrection ers_reference = quadCLTs[ref_index].getErsCorrection();
ErsCorrection ers_center = quadCLTs[cent_index].getErsCorrection();
String cent_ts = quadCLTs[cent_index].getImageName();
double [][] center_xyzatr = new double [][] {ers_reference.getSceneXYZ(cent_ts), ers_reference.getSceneATR(cent_ts)};
double [][] inv_cent_xyzatr = ErsCorrection.invertXYZATR(center_xyzatr);
// Invert half-sequence to reference cent_index
for (int scene_index = cent_index; scene_index <= ref_index; scene_index++) { // include cent_index itself to the map
double [][] scene_xyzatr,dxyzatr_dt;
if (scene_index == ref_index) {
......@@ -259,13 +305,17 @@ public class Interscene {
);
}
// set pointers
quadCLTs[ref_index].setRefPointer(cent_ts);
quadCLTs[cent_index].setFirstLastPointers(quadCLTs[earliest_scene2],quadCLTs[ref_index]);
quadCLTs[ref_index].set_orient(1); // first orientation // applicable to the center?
quadCLTs[ref_index].set_accum(0); // reset accumulations ("build_interscene") number
quadCLTs[ref_index].setRefPointer(cent_ts); // write pointer to center scene to reference scene
quadCLTs[cent_index].setFirstLastPointers(quadCLTs[earliest_scene2],quadCLTs[ref_index]);// set first/last to center scene
// set quadCLTs[ref_index].set_orient(0); // ? as it has only half?
// quadCLTs[ref_index].set_orient(1); // first orientation // applicable to the center?
// quadCLTs[ref_index].set_accum(0); // reset accumulations ("build_interscene") number
quadCLTs[ref_index].saveInterProperties( // save properties for interscene processing (extrinsics, ers, ...) // null pointer
null, // String path, // full name with extension or w/o path to use x3d directory
debugLevel+1);
quadCLTs[cent_index].set_orient(1); // first orientation
quadCLTs[cent_index].set_accum(0); // reset accumulations ("build_interscene") number
quadCLTs[cent_index].saveInterProperties( // save properties for interscene processing (extrinsics, ers, ...) // null pointer
......
......@@ -78,6 +78,7 @@ public class IntersceneMatchParameters {
ims_mount_atr[2] * Math.PI/180};
}
public boolean center_reference = false;
public boolean overlap_sequences = false; // overlap sequences: scan down from the previous center
public boolean reset_photometric = true; // reset photometric calibration - once for each new series
public boolean force_ref_dsi = false; // true;
public boolean force_orientations = false;
......@@ -527,6 +528,8 @@ public class IntersceneMatchParameters {
gd.addCheckbox ("Use center scene as a reference", this.center_reference,
"True for mapping with post-processing. If false, use the last scene as a reference (driving).");
gd.addCheckbox ("Overlap sequences", this.overlap_sequences,
"Overlap sequences by half - start (down) from the previous center.");
gd.addCheckbox ("Reset photometric calibration", this.reset_photometric,
"Reset photometric calibration, will use basic one before re-calibrating.");
gd.addCheckbox ("Force reference scene DSI calculation", this.force_ref_dsi,
......@@ -1228,6 +1231,7 @@ public class IntersceneMatchParameters {
this.sfm_extrapolate = gd.getNextBoolean();
this.center_reference = gd.getNextBoolean();
this.overlap_sequences = gd.getNextBoolean();
this.reset_photometric = gd.getNextBoolean();
this.force_ref_dsi = gd.getNextBoolean();
this.force_orientations = gd.getNextBoolean();
......@@ -1612,6 +1616,7 @@ public class IntersceneMatchParameters {
properties.setProperty(prefix+"sfm_extrapolate", this.sfm_extrapolate + ""); // boolean
properties.setProperty(prefix+"center_reference", this.center_reference + ""); // boolean
properties.setProperty(prefix+"overlap_sequences", this.overlap_sequences + ""); // boolean
properties.setProperty(prefix+"reset_photometric", this.reset_photometric + ""); // boolean
properties.setProperty(prefix+"force_ref_dsi", this.force_ref_dsi + ""); // boolean
properties.setProperty(prefix+"force_orientations", this.force_orientations + ""); // boolean
......@@ -1956,6 +1961,7 @@ public class IntersceneMatchParameters {
if (properties.getProperty(prefix+"sfm_extrapolate")!=null) this.sfm_extrapolate=Boolean.parseBoolean(properties.getProperty(prefix+"sfm_extrapolate"));
if (properties.getProperty(prefix+"center_reference")!=null) this.center_reference=Boolean.parseBoolean(properties.getProperty(prefix+"center_reference"));
if (properties.getProperty(prefix+"overlap_sequences")!=null) this.overlap_sequences=Boolean.parseBoolean(properties.getProperty(prefix+"overlap_sequences"));
if (properties.getProperty(prefix+"reset_photometric")!=null) this.reset_photometric=Boolean.parseBoolean(properties.getProperty(prefix+"reset_photometric"));
if (properties.getProperty(prefix+"force_ref_dsi")!=null) this.force_ref_dsi=Boolean.parseBoolean(properties.getProperty(prefix+"force_ref_dsi"));
if (properties.getProperty(prefix+"force_orientations")!=null) this.force_orientations=Boolean.parseBoolean(properties.getProperty(prefix+"force_orientations"));
......@@ -2327,6 +2333,7 @@ public class IntersceneMatchParameters {
imp.sfm_extrapolate = this.sfm_extrapolate;
imp.center_reference = this.center_reference;
imp.overlap_sequences = this.overlap_sequences;
imp.reset_photometric = this.reset_photometric;
imp.force_ref_dsi = this.force_ref_dsi;
imp.force_orientations = this.force_orientations;
......
......@@ -3966,6 +3966,7 @@ public class OpticalFlow {
public static void buildRefDSI(
CLTParameters clt_parameters,
boolean fast,
boolean skip_photo,
EyesisCorrectionParameters.DebayerParameters debayerParameters,
ColorProcParameters colorProcParameters,
EyesisCorrectionParameters.RGBParameters rgbParameters,
......@@ -4109,7 +4110,7 @@ public class OpticalFlow {
boolean ran_photo_each = false;
quadCLT_ref.tp.resetCLTPasses();
// perform photometric here, after first DSI
if (photo_each && (!quadCLT_ref.isPhotometricThis() || !batch_mode)) {
if (!skip_photo && (photo_each && (!quadCLT_ref.isPhotometricThis() || !batch_mode))) {
// if (debugLevel > -3) {
// System.out.println("**** Running photometric equalization for "+quadCLT_ref.getImageName()+
// ", current was from scene "+quadCLT_ref.getPhotometricScene()+" ****");
......@@ -4240,13 +4241,12 @@ public class OpticalFlow {
quadCLT_ref.saveDSIAll (
"-DSI_MAIN", // String suffix, // "-DSI_MAIN"
dsi);
quadCLT_ref.set_orient(0); // reset orientations
quadCLT_ref.set_accum(0); // reset accumulations ("build_interscene") number
quadCLT_ref.saveInterProperties( // save properties for interscene processing (extrinsics, ers, ...)
null, // String path, // full name with extension or w/o path to use x3d directory
// null, // Properties properties, // if null - will only save extrinsics)
debugLevel);
quadCLT_ref.set_orient(0); // reset orientations
quadCLT_ref.set_accum(0); // reset accumulations ("build_interscene") number
}
public static void reuseRefDSI(
......@@ -4760,6 +4760,7 @@ public class OpticalFlow {
scenes_xyzatr[last_index] = new double[2][3]; // all zeros
// See if build_ref_dsi is needed
if (!build_ref_dsi) {
// try reading full
quadCLTs[last_index] = (QuadCLT) quadCLT_main.spawnNoModelQuadCLT( // will conditionImageSet
set_channels[last_index].set_name,
clt_parameters,
......@@ -4771,10 +4772,24 @@ public class OpticalFlow {
null, // String path, // full name with extension or null to use x3d directory
false, // boolean all_properties,// null, // Properties properties, // if null - will only save extrinsics)
debugLevel);
int center_index =quadCLTs[last_index].getReferenceIndex(null);
if (center_index == -1) {
build_ref_dsi = true;
} else {
int center_index =quadCLTs[last_index].getReferenceIndex(new QuadCLT[] {quadCLTs[last_index]}); // null); self, may return 0
int [] first_last_index = quadCLTs[last_index].getFirstLastIndex(new QuadCLT[] {quadCLTs[last_index]});
if ((center_index == -1) || (center_index == 0)) { // 0 - was self-referencing, TODO: debug, should not be
// -2 - reference somewhere, not to itself
// -1 - no reference to a center
// 0 - reference
// with overlap it may already have *-DSI_MAIN or even *-INTER-INTRA-LMA.tiff
// int [] first_last_index = quadCLTs[last_index].getFirstLastIndex(quadCLTs);
if (first_last_index == null) {
build_ref_dsi = true;
} else { // should have DSI
if (debugLevel >-2) {
System.out.println("no link to center, but this seems to be other's center "+quadCLTs[last_index].getReferenceTimestamp()+
", forcing initial orientation.");
}
force_initial_orientations = true;
}
} else { // -2 - reference somewhere, not to itself
QuadCLT try_ref_scene = (QuadCLT) quadCLT_main.spawnQuadCLT( // will conditionImageSet
quadCLTs[last_index].getReferenceTimestamp(), // set_channels[last_index].set_name,
clt_parameters,
......@@ -4785,6 +4800,7 @@ public class OpticalFlow {
if (debugLevel >-2) {
System.out.println("DSI data for scene "+quadCLTs[last_index].getReferenceTimestamp()+
" does not exist, forcing initial orientation.");
// consider copyJP4src for the lower half (now it is not needed)
}
force_initial_orientations = true;
}
......@@ -4848,6 +4864,7 @@ public class OpticalFlow {
buildRefDSI( // returned is a different instance than input -FIXED
clt_parameters, // CLTParameters clt_parameters,
fast, // boolean fast,
false, // boolean skip_photo,
debayerParameters, // EyesisCorrectionParameters.DebayerParameters debayerParameters,
colorProcParameters, // ColorProcParameters colorProcParameters,
rgbParameters, // EyesisCorrectionParameters.RGBParameters rgbParameters,
......@@ -4884,7 +4901,7 @@ public class OpticalFlow {
updateStatus, // final boolean updateStatus,
debugLevel); // int debugLevel)
if (center_reference) {
// center_ts = getReferenceTimestamp();
// center_ts = getReferenceTimestamp(); //??
}
}
} // while (blue_sky == null)
......@@ -5054,7 +5071,11 @@ public class OpticalFlow {
int [] first_last_index = quadCLTs[ref_index].getFirstLastIndex(quadCLTs);
if (first_last_index == null) {
System.out.println("BUG: saved reference scene "+quadCLTs[ref_index].getImageName()+
" does not have first/last scenes defined");
" is not really a reference");
System.out.println();
} else if ((first_last_index [0] < 0) || (first_last_index [1] < 0)) {
System.out.println("BUG: saved reference scene "+quadCLTs[ref_index].getImageName()+
" does not have both first/last scenes defined: ["+first_last_index [0]+", "+first_last_index [1]+"]");
System.out.println();
} else {
earliest_scene = first_last_index[0];
......@@ -5100,17 +5121,18 @@ public class OpticalFlow {
// num_avg_pairs = last_index - ref_index;
// }
QuadCLT[][][] scenes_seq_pairs = new QuadCLT[3][num_avg_pairs][2];
int ref_index_mod1 = Math.max(ref_index, earliest_scene + num_avg_pairs - 1);
int ref_index_mod2 = Math.min(ref_index, last_index);
// TODO: calculate horizontal offset and compare with sfm_min_base
for (int i = 0; i < num_avg_pairs; i++) {
// scenes_seq_pairs[0][i][0] = quadCLTs[ref_index + num_avg_pairs - 1 - i];
scenes_seq_pairs[0][i][0] = quadCLTs[last_index - i];
scenes_seq_pairs[0][i][1] = quadCLTs[earliest_scene + num_avg_pairs - 1 - i];
scenes_seq_pairs[1][i][0] = quadCLTs[last_index - i];
scenes_seq_pairs[1][i][1] = quadCLTs[ref_index - i];
scenes_seq_pairs[1][i][1] = quadCLTs[ref_index_mod1 - i];
scenes_seq_pairs[2][i][0] = quadCLTs[last_index - i];
scenes_seq_pairs[2][i][0] = quadCLTs[ref_index_mod2 + num_avg_pairs - 1 - i];
scenes_seq_pairs[2][i][1] = quadCLTs[earliest_scene + num_avg_pairs - 1 - i];
}
double [][] sfm_dsn = StructureFromMotion.sfmPairsSet(
......
......@@ -273,7 +273,7 @@ public class QuadCLTCPU {
public int getReferenceIndex(QuadCLTCPU [] scenes) {
if (timestamp_reference != null) {
if (scenes != null) {
for (int i = 0; i < scenes.length; i++) {
for (int i = 0; i < scenes.length; i++) if (scenes[i] != null){
if (timestamp_reference.equals(scenes[i].getImageName())) {
return i;
}
......@@ -286,33 +286,36 @@ public class QuadCLTCPU {
/**
* If this scene is a reference scene, then return array of {first_index, last_index}.
* If this scene is not a reference or the array does not contain both the first and
* the last scene - return null.
* If this scene is not a reference - return null.
* @param scenes array of scenes (QuadCLTCPU instances)
* @return a pair of {first_index, last_index}
*/
public int [] getFirstLastIndex(QuadCLTCPU [] scenes) { // should be ordered accending ts
int [] fl = new int [] {-1,-1};
if ((timestamp_first != null) && (timestamp_last != null)) {
int i = 0;
for (; i < scenes.length; i++) {
if (timestamp_first.equals(scenes[i].getImageName())) {
fl[0] = i;
break;
}
}
for (; i < scenes.length; i++) {
if (timestamp_last.equals(scenes[i].getImageName())) {
fl[1] = i;
break;
}
}
}
if ((fl[0] >= 0) && (fl[1] >= 0)) {
return fl;
} else {
return null;
}
int [] fl = null;
if ((timestamp_first != null) && (timestamp_last != null)) {
fl = new int [] {-1,-1};
if (scenes != null) {
int i = 0;
for (; i < scenes.length; i++) if (scenes[i] != null) {
if (timestamp_first.equals(scenes[i].getImageName())) {
fl[0] = i;
break;
}
}
for (; i < scenes.length; i++) if (scenes[i] != null) {
if (timestamp_last.equals(scenes[i].getImageName())) {
fl[1] = i;
break;
}
}
}
}
return fl;
// if ((fl[0] >= 0) && (fl[1] >= 0)) {
// return fl;
// } else {
// return null;
// }
}
/**
......
......@@ -8531,6 +8531,7 @@ if (debugLevel > -100) return true; // temporarily !
final boolean updateStatus,
final int debugLevel) throws Exception
{
boolean overlap_sequences = clt_parameters.imp.overlap_sequences;
int stereo_gap = clt_parameters.imp.stereo_gap;
double stereo_intereye = clt_parameters.imp.stereo_intereye;
double stereo_phone_width = clt_parameters.imp.stereo_phone_width; // 0 - no padding
......@@ -8669,17 +8670,13 @@ if (debugLevel > -100) return true; // temporarily !
}
Files.createSymbolicLink(link.toPath(), pathRelative);
}
if (start_ref_pointers[0] < (min_num_scenes-1)) {
if (start_ref_pointers[0] < (min_num_scenes-1)) {
break;
}
ref_index = start_ref_pointers[0]; // continue from the same attached to the previous reference
ref_index = start_ref_pointers[overlap_sequences? 1 : 0]; // continue from the same attached to the previous reference
}
}
// combine videos if generated
if ((video_sets_list.size() > 1) &&
(video_sets_list.get(0).getVideoPaths() != null) &&
......
......@@ -948,28 +948,40 @@ public class StructureFromMotion {
}
}
// check each pair has sufficient baseline and contains exactly 2 scenes
int num_good_sets=0;
boolean [] good_set = new boolean [num_sets];
for (int nset=0; nset < num_sets; nset++) {
int num_pairs = scene_pairs_sets[nset].length;
for (int npair = 0; npair < num_pairs; npair++) {
double [][][] scenes_xyzatr= scenes_xyzatr_sets[nset][npair];
if (scenes_xyzatr.length != 2) {
System.out.println("sfmPairsSet(): BUG: Not a pair of scenes: scenes_xyzatr_sets["+nset+"]["+npair+
"].length = "+(scenes_xyzatr.length)+" != 2");
return null;
}
double dx = scenes_xyzatr[1][0][0] - scenes_xyzatr[0][0][0];
double dy = scenes_xyzatr[1][0][1] - scenes_xyzatr[0][0][1];
double base = Math.sqrt(dx*dx + dy*dy);
if (base < sfm_min_base) {
if (debugLevel > -3) {
System.out.println("sfmPairsSet(): stereo base for nset="+nset+", npair="+npair+
" = "+base+" < "+sfm_min_base+", bailing out from SfM ranging.");
check_set: {
for (int npair = 0; npair < num_pairs; npair++) {
double [][][] scenes_xyzatr= scenes_xyzatr_sets[nset][npair];
if (scenes_xyzatr.length != 2) {
System.out.println("sfmPairsSet(): BUG: Not a pair of scenes: scenes_xyzatr_sets["+nset+"]["+npair+
"].length = "+(scenes_xyzatr.length)+" != 2");
return null;
}
double dx = scenes_xyzatr[1][0][0] - scenes_xyzatr[0][0][0];
double dy = scenes_xyzatr[1][0][1] - scenes_xyzatr[0][0][1];
double base = Math.sqrt(dx*dx + dy*dy);
if (base < sfm_min_base) {
if (debugLevel > -3) {
System.out.println("sfmPairsSet(): stereo base for nset="+nset+", npair="+npair+
" = "+base+" < "+sfm_min_base+".");
}
break check_set; // return null;
}
return null;
}
good_set[nset] = true;
num_good_sets++;
}
}
if (num_good_sets == 0) {
if (debugLevel > -3) {
System.out.println("sfmPairsSet(): No good sets with sufficient base found, bailing out from SfM ranging.");
}
}
for (int ntry = 0; ntry < num_readjust; ntry++) {
final SfmCorr [] sfmCorrCombo = new SfmCorr [tilesX*tilesY];
double [] sum_weights_ind= new double [tilesX*tilesY];
......@@ -982,9 +994,7 @@ public class StructureFromMotion {
*/
final int [] corr_mode = new int [tilesX*tilesY];
for (int nset=0; nset < num_sets; nset++) {
for (int nset=0; nset < num_sets; nset++) if (good_set[nset]){
SfmCorr [] sfmCorr = getSfmCorr(
clt_parameters, // final CLTParameters clt_parameters,
ref_scene, // final QuadCLT ref_scene,
......@@ -1009,7 +1019,7 @@ public class StructureFromMotion {
if (debugLevel > -3) {
System.out.println("sfmPairsSet(): not enough SfM tiles, bailing out");
}
return null;
continue; //return null;
}
if ((sfm_shrink > 0) || (fade_sigma > 0.0)) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment