From 15cff9c771980dfb234f17135d05e711fcd64c70 Mon Sep 17 00:00:00 2001 From: AndreyFilippov Date: Mon, 9 Nov 2020 19:56:46 -0700 Subject: [PATCH] Generated statistics from mixed synthetic noise --- .../elphel/imagej/cameras/CLTParameters.java | 39 +- .../imagej/cameras/InterNoiseParameters.java | 91 ++++ .../imagej/correction/Eyesis_Correction.java | 351 +++++++++++++- .../elphel/imagej/tileprocessor/ImageDtt.java | 6 +- .../IntersceneLmaParameters.java | 24 + .../imagej/tileprocessor/OpticalFlow.java | 377 +++++++++++---- .../elphel/imagej/tileprocessor/QuadCLT.java | 343 ++++++++++++-- .../imagej/tileprocessor/QuadCLTCPU.java | 45 +- .../imagej/tileprocessor/TwoQuadCLT.java | 446 +++++++++++++++++- 9 files changed, 1561 insertions(+), 161 deletions(-) create mode 100644 src/main/java/com/elphel/imagej/cameras/InterNoiseParameters.java diff --git a/src/main/java/com/elphel/imagej/cameras/CLTParameters.java b/src/main/java/com/elphel/imagej/cameras/CLTParameters.java index f08bd0d..cd2393c 100644 --- a/src/main/java/com/elphel/imagej/cameras/CLTParameters.java +++ b/src/main/java/com/elphel/imagej/cameras/CLTParameters.java @@ -1,4 +1,32 @@ package com.elphel.imagej.cameras; +/** + ** + ** CLTParameters - Class for handling multiple configuration parameters + ** + ** Copyright (C) 2017-2020 Elphel, Inc. + ** + ** -----------------------------------------------------------------------------** + ** + ** CLTParameters.java is free software: you can redistribute it and/or modify + ** it under the terms of the GNU General Public License as published by + ** the Free Software Foundation, either version 3 of the License, or + ** (at your option) any later version. + ** + ** This program is distributed in the hope that it will be useful, + ** but WITHOUT ANY WARRANTY; without even the implied warranty of + ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + ** GNU General Public License for more details. + ** + ** You should have received a copy of the GNU General Public License + ** along with this program. If not, see . + ** -----------------------------------------------------------------------------** + ** + */ + +//import static jcuda.driver.JCudaDriver.cuMemcpyDtoH; + + + import java.util.ArrayList; import java.util.Collections; @@ -875,8 +903,8 @@ public class CLTParameters { public LwirReaderParameters lwir = new LwirReaderParameters(); public OpticalFlowParameters ofp = new OpticalFlowParameters(); public IntersceneLmaParameters ilp = new IntersceneLmaParameters(); - - + public InterNoiseParameters inp = new InterNoiseParameters(); + public HashMap z_corr_map = new HashMap(); //old one public HashMap infinity_distace_map = new HashMap(); //new one public static String Z_CORR_PREFIX = "z_corr."; @@ -1729,6 +1757,7 @@ public class CLTParameters { lwir.setProperties (prefix+"_lwir", properties); ofp.setProperties (prefix+"_ofp_", properties); ilp.setProperties (prefix+"_ilp_", properties); + inp.setProperties (prefix+"_inp_", properties); } @@ -2554,6 +2583,7 @@ public class CLTParameters { lwir.getProperties (prefix+"_lwir", properties); ofp.getProperties (prefix+"_ofp_", properties); ilp.getProperties (prefix+"_ilp_", properties); + inp.getProperties (prefix+"_inp_", properties); } public boolean showJDialog() { @@ -3528,9 +3558,11 @@ public class CLTParameters { gd.addTab ("O-Flow", "parameters for the interscene Optical FLow calculations"); this.ofp.dialogQuestions(gd); - gd.addTab ("Intra-LMA", "parameters for the interscene LMA fitting"); + gd.addTab ("Inter-LMA", "parameters for the interscene LMA fitting"); this.ilp.dialogQuestions(gd); + gd.addTab ("Inter-Noise", "parameters for the interscene noise testing"); + this.inp.dialogQuestions(gd); gd.addTab ("Debug", "Other debug images"); gd.addMessage ("--- Other debug images ---"); @@ -4330,6 +4362,7 @@ public class CLTParameters { this.lwir.dialogAnswers(gd); this.ofp.dialogAnswers(gd); this.ilp.dialogAnswers(gd); + this.inp.dialogAnswers(gd); this.debug_initial_discriminate= gd.getNextBoolean(); this.dbg_migrate= gd.getNextBoolean(); diff --git a/src/main/java/com/elphel/imagej/cameras/InterNoiseParameters.java b/src/main/java/com/elphel/imagej/cameras/InterNoiseParameters.java new file mode 100644 index 0000000..3047494 --- /dev/null +++ b/src/main/java/com/elphel/imagej/cameras/InterNoiseParameters.java @@ -0,0 +1,91 @@ +package com.elphel.imagej.cameras; +/** + ** + ** InterNoiseParameters - Class for handling configuration parameters + ** to measure disparity map calculation with images degraded by the artificially + ** introduced noise. + ** related to the interscene LMA + ** + ** Copyright (C) 2020 Elphel, Inc. + ** + ** -----------------------------------------------------------------------------** + ** + ** InterNoiseParameters.java is free software: you can redistribute it and/or modify + ** it under the terms of the GNU General Public License as published by + ** the Free Software Foundation, either version 3 of the License, or + ** (at your option) any later version. + ** + ** This program is distributed in the hope that it will be useful, + ** but WITHOUT ANY WARRANTY; without even the implied warranty of + ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + ** GNU General Public License for more details. + ** + ** You should have received a copy of the GNU General Public License + ** along with this program. If not, see . + ** -----------------------------------------------------------------------------** + ** + */ +import java.util.Properties; + +import com.elphel.imagej.common.GenericJTabbedDialog; + + +public class InterNoiseParameters { + public double noise_sigma = 1.5; + public double noise_scale = 0.01; + public double initial_offset = 1.0; + public boolean ref_only = false; // also see imgdtt_params.dbg_pair_mask to switch between all pairs (63) and binocular only (1) + public int noise_debug_level = -1; // Noise testing debug level + + + public void dialogQuestions(GenericJTabbedDialog gd) { + gd.addMessage("Additive noise parameters"); + gd.addMessage("LMA other parameters"); + gd.addNumericField("Noise Gaussian sigma", this.noise_sigma, 3,5,"pix", + "Blur noise with 2D Gaussian"); + gd.addNumericField("Scale noise", this.noise_scale, 6,8,"", + "Scale noise relative to the average value of the color component."); + gd.addNumericField("Offset target disparity", this.initial_offset, 3,5,"pix", + "Offset target disparity before attempting to correlate and refine."); + gd.addCheckbox ("Reference scene only", this.ref_only, + "Process only reference scene (intra-scene, no inter-scene accumulation)."); + gd.addNumericField("Debug level", this.noise_debug_level, 0,3,"", + "Debug level of interscene noise testing."); + + } + public void dialogAnswers(GenericJTabbedDialog gd) { + this.noise_sigma = gd.getNextNumber(); + this.noise_scale = gd.getNextNumber(); + this.initial_offset = gd.getNextNumber(); + this.ref_only = gd.getNextBoolean(); + this.noise_debug_level = (int) gd.getNextNumber(); + } + + public void setProperties(String prefix,Properties properties){ + properties.setProperty(prefix+"noise_sigma", this.noise_sigma+""); + properties.setProperty(prefix+"noise_scale", this.noise_scale+""); + properties.setProperty(prefix+"initial_offset", this.initial_offset+""); + properties.setProperty(prefix+"ref_only", this.ref_only+""); + properties.setProperty(prefix+"noise_debug_level", this.noise_debug_level+""); + } + public void getProperties(String prefix,Properties properties){ + if (properties.getProperty(prefix+"noise_sigma")!=null) this.noise_sigma=Double.parseDouble(properties.getProperty(prefix+"noise_sigma")); + if (properties.getProperty(prefix+"noise_scale")!=null) this.noise_scale=Double.parseDouble(properties.getProperty(prefix+"noise_scale")); + if (properties.getProperty(prefix+"initial_offset")!=null) this.initial_offset=Double.parseDouble(properties.getProperty(prefix+"initial_offset")); + if (properties.getProperty(prefix+"ref_only")!=null) this.ref_only=Boolean.parseBoolean(properties.getProperty(prefix+"ref_only")); + if (properties.getProperty(prefix+"noise_debug_level")!=null) this.noise_debug_level=Integer.parseInt(properties.getProperty(prefix+"noise_debug_level")); + } + @Override + public InterNoiseParameters clone() throws CloneNotSupportedException { + InterNoiseParameters inp = new InterNoiseParameters(); + inp.noise_sigma = this.noise_sigma; + inp.noise_scale = this.noise_scale; + inp.initial_offset = this.initial_offset; + inp.ref_only = this.ref_only; + inp.noise_debug_level = this.noise_debug_level; + return inp; + } + + + +} diff --git a/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java b/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java index 2e680f7..93d99a0 100644 --- a/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java +++ b/src/main/java/com/elphel/imagej/correction/Eyesis_Correction.java @@ -59,6 +59,7 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; @@ -75,6 +76,7 @@ import com.elphel.imagej.calibration.PixelMapping; import com.elphel.imagej.cameras.CLTParameters; import com.elphel.imagej.cameras.ColorProcParameters; import com.elphel.imagej.cameras.EyesisCorrectionParameters; +import com.elphel.imagej.cameras.ThermalColor; import com.elphel.imagej.common.DoubleFHT; import com.elphel.imagej.common.DoubleGaussianBlur; import com.elphel.imagej.common.GenericJTabbedDialog; @@ -704,6 +706,9 @@ private Panel panel1, addButton("Inter LMA", panelClt5, color_stop); addButton("Inter Series", panelClt5, color_process); addButton("Inter Accumulate", panelClt5, color_process); + addButton("Inter Noise", panelClt5, color_process); + addButton("Noise Stats", panelClt5, color_process); + addButton("Colorize Depth", panelClt5, color_process); plugInFrame.add(panelClt5); } @@ -5126,13 +5131,36 @@ private Panel panel1, interSeriesLMA(); return; - /* ======================================================================== */ +/* ======================================================================== */ } else if (label.equals("Inter Accumulate")) { DEBUG_LEVEL=MASTER_DEBUG_LEVEL; EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL); CLT_PARAMETERS.batch_run = true; intersceneAccumulate(); return; + +/* ======================================================================== */ + } else if (label.equals("Inter Noise")) { + DEBUG_LEVEL=MASTER_DEBUG_LEVEL; + EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL); + CLT_PARAMETERS.batch_run = true; + intersceneNoise(); + return; + +/* ======================================================================== */ + } else if (label.equals("Noise Stats")) { + DEBUG_LEVEL=MASTER_DEBUG_LEVEL; + EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL); + CLT_PARAMETERS.batch_run = true; + intersceneNoiseStats(); + return; +/* ======================================================================== */ + } else if (label.equals("Colorize Depth")) { + DEBUG_LEVEL=MASTER_DEBUG_LEVEL; + EYESIS_CORRECTIONS.setDebug(DEBUG_LEVEL); + CLT_PARAMETERS.batch_run = true; + coloriseDepthMap(); + return; /* ======================================================================== */ } else if (label.equals("Inter LMA")) { @@ -6793,6 +6821,327 @@ private Panel panel1, return true; } + public boolean intersceneNoise() { + long startTime=System.nanoTime(); + // load needed sensor and kernels files + if (!prepareRigImages()) return false; + String configPath=getSaveCongigPath(); + if (configPath.equals("ABORT")) return false; + setAllProperties(PROPERTIES); // batchRig may save properties with the model. Extrinsics will be updated, others should be set here + if (DEBUG_LEVEL > -2){ + System.out.println("++++++++++++++ Testing Interscene processing ++++++++++++++"); + } + + if (CLT_PARAMETERS.useGPU()) { // only init GPU instances if it is used + if (GPU_TILE_PROCESSOR == null) { + try { + GPU_TILE_PROCESSOR = new GPUTileProcessor(CORRECTION_PARAMETERS.tile_processor_gpu); + } catch (Exception e) { + System.out.println("Failed to initialize GPU class"); + // TODO Auto-generated catch block + e.printStackTrace(); + return false; + } //final int debugLevel); + } + if (CLT_PARAMETERS.useGPU(false) && (QUAD_CLT != null) && (GPU_QUAD == null)) { // if GPU main is needed + try { + GPU_QUAD = GPU_TILE_PROCESSOR.new GpuQuad( + QUAD_CLT, + 4, + 3); + } catch (Exception e) { + System.out.println("Failed to initialize GpuQuad class"); + // TODO Auto-generated catch block + e.printStackTrace(); + return false; + } //final int debugLevel); + QUAD_CLT.setGPU(GPU_QUAD); + } + } + + try { + TWO_QUAD_CLT.intersceneNoise( + QUAD_CLT, // QuadCLT quadCLT_main, + CLT_PARAMETERS, // EyesisCorrectionParameters.DCTParameters dct_parameters, + DEBAYER_PARAMETERS, //EyesisCorrectionParameters.DebayerParameters debayerParameters, + COLOR_PROC_PARAMETERS, //EyesisCorrectionParameters.ColorProcParameters colorProcParameters, + CHANNEL_GAINS_PARAMETERS, //CorrectionColorProc.ColorGainsParameters channelGainParameters, + RGB_PARAMETERS, //EyesisCorrectionParameters.RGBParameters rgbParameters, + EQUIRECTANGULAR_PARAMETERS, // EyesisCorrectionParameters.EquirectangularParameters equirectangularParameters, + PROPERTIES, // Properties properties, + THREADS_MAX, //final int threadsMax, // maximal number of threads to launch + UPDATE_STATUS, //final boolean updateStatus, + DEBUG_LEVEL); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } //final int debugLevel); + if (configPath!=null) { + saveTimestampedProperties( // save config again + configPath, // full path or null + null, // use as default directory if path==null + true, + PROPERTIES); + } + System.out.println("batchRig(): Processing finished at "+ + IJ.d2s(0.000000001*(System.nanoTime()-startTime),3)+" sec, --- Free memory="+ + Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); + return true; + } + + public boolean intersceneNoiseStats() { + long startTime=System.nanoTime(); + // load needed sensor and kernels files + if (!prepareRigImages()) return false; + String configPath=getSaveCongigPath(); + if (configPath.equals("ABORT")) return false; + setAllProperties(PROPERTIES); // batchRig may save properties with the model. Extrinsics will be updated, others should be set here + if (DEBUG_LEVEL > -2){ + System.out.println("++++++++++++++ Testing Interscene processing ++++++++++++++"); + } + + if (CLT_PARAMETERS.useGPU()) { // only init GPU instances if it is used + if (GPU_TILE_PROCESSOR == null) { + try { + GPU_TILE_PROCESSOR = new GPUTileProcessor(CORRECTION_PARAMETERS.tile_processor_gpu); + } catch (Exception e) { + System.out.println("Failed to initialize GPU class"); + // TODO Auto-generated catch block + e.printStackTrace(); + return false; + } //final int debugLevel); + } + if (CLT_PARAMETERS.useGPU(false) && (QUAD_CLT != null) && (GPU_QUAD == null)) { // if GPU main is needed + try { + GPU_QUAD = GPU_TILE_PROCESSOR.new GpuQuad( + QUAD_CLT, + 4, + 3); + } catch (Exception e) { + System.out.println("Failed to initialize GpuQuad class"); + // TODO Auto-generated catch block + e.printStackTrace(); + return false; + } //final int debugLevel); + QUAD_CLT.setGPU(GPU_QUAD); + } + } + + try { + TWO_QUAD_CLT.intersceneNoiseStats( + QUAD_CLT, // QuadCLT quadCLT_main, + CLT_PARAMETERS, // EyesisCorrectionParameters.DCTParameters dct_parameters, + DEBAYER_PARAMETERS, //EyesisCorrectionParameters.DebayerParameters debayerParameters, + COLOR_PROC_PARAMETERS, //EyesisCorrectionParameters.ColorProcParameters colorProcParameters, + CHANNEL_GAINS_PARAMETERS, //CorrectionColorProc.ColorGainsParameters channelGainParameters, + RGB_PARAMETERS, //EyesisCorrectionParameters.RGBParameters rgbParameters, + EQUIRECTANGULAR_PARAMETERS, // EyesisCorrectionParameters.EquirectangularParameters equirectangularParameters, + PROPERTIES, // Properties properties, + THREADS_MAX, //final int threadsMax, // maximal number of threads to launch + UPDATE_STATUS, //final boolean updateStatus, + DEBUG_LEVEL); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } //final int debugLevel); + if (configPath!=null) { + saveTimestampedProperties( // save config again + configPath, // full path or null + null, // use as default directory if path==null + true, + PROPERTIES); + } + System.out.println("batchRig(): Processing finished at "+ + IJ.d2s(0.000000001*(System.nanoTime()-startTime),3)+" sec, --- Free memory="+ + Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); + return true; + } + + public boolean coloriseDepthMap() { + ImagePlus imp_sel = WindowManager.getCurrentImage(); + if (imp_sel==null){ + IJ.showMessage("Error","No images selected"); + return false; + } + double disparity0 = 0.75; + double disparity_max = 75.0; // pix + boolean show_dialog2 = false; + boolean linear_disparity = false; + + int legend_width = 0; + int legend_gap = 5; + GenericJTabbedDialog gd0 = new GenericJTabbedDialog("Ln mode"); + gd0.addNumericField("disparity0", disparity0, 5,8,"pix", "Disparity to swict from linear to log. ) - skip log mode"); + gd0.addNumericField("Maximal disparity", disparity_max, 5,8,"pix", "Fixed maximal disparity (0 - auto"); + gd0.addCheckbox("Linear disparity legend", linear_disparity); + gd0.addNumericField("Legend width", legend_width, 0,3,"", "Optional disparity legend vertical bar width"); + gd0.addNumericField("Legend gap", legend_gap, 0,3,"", "Optional disparity legend vertical bar gap"); + gd0.addCheckbox("Show second dialog", show_dialog2); + + gd0.showDialog(); + if (gd0.wasCanceled()) return false; + disparity0= gd0.getNextNumber(); + disparity_max= gd0.getNextNumber(); + linear_disparity = gd0.getNextBoolean(); + legend_width= (int) gd0.getNextNumber(); + legend_gap= (int) gd0.getNextNumber(); + show_dialog2 = gd0.getNextBoolean(); + + boolean log_mode = disparity0 > 0.0; + + + int current_slice = imp_sel.getCurrentSlice(); + ImageStack imageStack = imp_sel.getStack(); + float [] fpixels = (float[]) imageStack.getPixels(current_slice); + int width = imp_sel.getWidth(); + int height = imp_sel.getHeight(); + String title = imp_sel.getShortTitle(); // getTitle(); + double [] dpixels; + int width0 = width; + if (legend_width <=0 ) { + dpixels= new double [fpixels.length]; + for (int i = 0; i < fpixels.length; i++) { + double d = fpixels[i]; + dpixels[i] = d; + } + } else { + width += legend_width + legend_gap; + dpixels= new double [height * width]; + Arrays.fill(dpixels, Double.NaN); + double mx = disparity_max; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width0; x++) { + double d = fpixels[ x + y * width0]; + dpixels[x + y * width] = d; + if (d > mx) { + mx = d; + } + } + } + if (disparity_max > 0) { + mx = disparity_max; + } + if (linear_disparity) { + for (int y = 0; y < height; y++) { + double d = (mx * y) / height; + for (int i = 0; i < legend_width; i++) { + dpixels[width0 + legend_gap + i + y * width] = d; + } + } + } + } + double mn = fpixels[0]; + double mx = mn; + double pwr = 1.0; + int palette = 2; + for (int i = 0; i < fpixels.length; i++) { + double d = dpixels[i]; + if (log_mode) { + if (!Double.isNaN(d)) { + if (d < 0.0) { // + d = 0.0; + } else if (d < disparity0) { + d = d/disparity0; + } else { + d = Math.log(d/disparity0) + 1.0; + } + } + } + if (!Double.isNaN(d)) { + if (!(d <= mx)) mx = d; + if (!(d >= mn)) mn = d; + } + dpixels[i] = d; + } + if (disparity_max > 0) { + mn = 0.0; + double d = disparity_max; + if (d < 0.0) { // + d = 0.0; + } else if (d < disparity0) { + d = d/disparity0; + } else { + d = Math.log(d/disparity0) + 1.0; + } + mx = d; + + } + + if (show_dialog2) { + GenericJTabbedDialog gd = new GenericJTabbedDialog("Colorization"+(log_mode?" {log mode)":"")); + gd.addNumericField("min", mn, 5,8,"", "Minimal value to map"); + gd.addNumericField("max", mx, 5,8,"", "Maximal value to map"); + gd.addNumericField("pwr", pwr, 5,8,"", "Exponent power"); + gd.addNumericField("palette", palette, 0,3,"", "Palette index"); + gd.showDialog(); + if (gd.wasCanceled()) return false; + mn= gd.getNextNumber(); + mx= gd.getNextNumber(); + pwr= gd.getNextNumber(); + palette= (int)gd.getNextNumber(); + } + if (pwr != 1.0) { + if (mn < 0) { + mn = 0.0; + } + mn = Math.pow(mn, pwr); + mx = Math.pow(mx, pwr); + for (int i = 0; i < dpixels.length; i++) { + if (dpixels[i] < 0) { + dpixels[i] = 0.0; + + } else { + dpixels[i] = Math.pow(dpixels[i], pwr); + } + } + } + + if (!linear_disparity && (legend_width > 0) ) { + for (int y = 0; y < height; y++) { + double d = ((mx-mn) * y) / height; + for (int i = 0; i < legend_width; i++) { + dpixels[width0 + legend_gap + i + y * width] = d; + } + } + } + + + double [][] pseudo_pixels = new double [3] [dpixels.length]; + ThermalColor tc = new ThermalColor( + palette, // public int lwir_palette = 0; // 0 - white - hot, 1 - black - hot, 2+ - colored + mn, + mx, + 255.0); + for (int i = 0; i < dpixels.length; i++) { + double [] rgb = tc.getRGB(dpixels[i]); + pseudo_pixels[0][i] = rgb[0]; // red + pseudo_pixels[1][i] = rgb[1]; // green + pseudo_pixels[2][i] = rgb[2]; // blue + } + String [] rgb_titles = {"red","green","blue"}; + + ImageStack stack = (new ShowDoubleFloatArrays()).makeStack( + pseudo_pixels, // iclt_data, + width, // (tilesX + 0) * clt_parameters.transform_size, + height, // (tilesY + 0) * clt_parameters.transform_size, + rgb_titles, // or use null to get chn-nn slice names + true); // replace NaN with 0.0 + ImagePlus imp_pseudo = EyesisCorrections.convertRGBAFloatToRGBA32( + stack, // ImageStack stackFloat, //r,g,b,a + // name+"ARGB"+suffix, // String title, + title+"-pseudo", // String title, + 0.0, // double r_min, + 255.0, // double r_max, + 0.0, // double g_min, + 255.0, // double g_max, + 0.0, // double b_min, + 255.0, // double b_max, + 0.0, // double alpha_min, + 1.0); // double alpha_max) + imp_pseudo.getProcessor().resetMinAndMax(); + imp_pseudo.show(); + return true; + } public boolean exportMLData() { diff --git a/src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java b/src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java index 9ef8d33..18c9264 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/ImageDtt.java @@ -1487,7 +1487,7 @@ public class ImageDtt extends ImageDttCPU { for (int i = 0; i < debug_offsets.length; i++) for (int j = 0; j < debug_offsets[i].length; j++) { debug_offsets[i][j] = imgdtt_params.lma_dbg_offset[i][j]*imgdtt_params.lma_dbg_scale; } - +//dbg_pair_mask final int quad = 4; // number of subcameras // final int numcol = isMonochrome()?1:3; @@ -1723,9 +1723,9 @@ public class ImageDtt extends ImageDttCPU { if (fcorr_tiles != null) { fcorr_tiles[tileY * tilesX + tileX] = fcorrs; // does not require corr_common_GPU() } - if ((disparity_map != null) || (clt_corr_partial != null) || (clt_mismatch != null)) { - int used_pairs = pair_mask; // imgdtt_params.dbg_pair_mask; //TODO: use tile tasks +// int used_pairs = pair_mask; // imgdtt_params.dbg_pair_mask; //TODO: use tile tasks + int used_pairs = pair_mask & imgdtt_params.dbg_pair_mask; // imgdtt_params.dbg_pair_mask; //TODO: use tile tasks int tile_lma_debug_level = ((tileX == debug_tileX) && (tileY == debug_tileY))? (imgdtt_params.lma_debug_level-1) : -2; boolean debugTile =(tileX == debug_tileX) && (tileY == debug_tileY) && (globalDebugLevel > -1); corr_common_GPU( diff --git a/src/main/java/com/elphel/imagej/tileprocessor/IntersceneLmaParameters.java b/src/main/java/com/elphel/imagej/tileprocessor/IntersceneLmaParameters.java index 56a60b0..93638da 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/IntersceneLmaParameters.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/IntersceneLmaParameters.java @@ -1,4 +1,28 @@ package com.elphel.imagej.tileprocessor; +/** + ** + ** IntersceneLmaParameters - Class for handling multiple configuration parameters + ** related to the interscene LMA + ** + ** Copyright (C) 2020 Elphel, Inc. + ** + ** -----------------------------------------------------------------------------** + ** + ** IntersceneLmaParameters.java is free software: you can redistribute it and/or modify + ** it under the terms of the GNU General Public License as published by + ** the Free Software Foundation, either version 3 of the License, or + ** (at your option) any later version. + ** + ** This program is distributed in the hope that it will be useful, + ** but WITHOUT ANY WARRANTY; without even the implied warranty of + ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + ** GNU General Public License for more details. + ** + ** You should have received a copy of the GNU General Public License + ** along with this program. If not, see . + ** -----------------------------------------------------------------------------** + ** + */ import java.util.Properties; diff --git a/src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java b/src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java index db8df94..740eb05 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java @@ -2263,7 +2263,7 @@ public class OpticalFlow { scene_atr, // final double [] scene_atr, // camera orientation relative to world frame scene_QuadClt, // final QuadCLT scene_QuadClt, reference_QuadClt); // final QuadCLT reference_QuadClt) - scene_QuadClt.getGpuQuad().setInterTasks( + scene_QuadClt.getGPU().setInterTasks( scene_pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls) scene_QuadClt.getGeometryCorrection(), // final GeometryCorrection geometryCorrection, scene_disparity_cor, // final double disparity_corr, @@ -3114,10 +3114,11 @@ public class OpticalFlow { } public void IntersceneAccumulate( - CLTParameters clt_parameters, - ColorProcParameters colorProcParameters, - QuadCLT ref_scene, // ordered by increasing timestamps - int debug_level + CLTParameters clt_parameters, + ColorProcParameters colorProcParameters, + QuadCLT ref_scene, // ordered by increasing timestamps + double [] noise_sigma_level, + int debug_level ) { System.out.println("IntersceneAccumulate(), scene timestamp="+ref_scene.getImageName()); @@ -3130,10 +3131,11 @@ public class OpticalFlow { scenes[indx_ref] = ref_scene; for (int i = 0; i < sts.length; i++) { - scenes[i] = ref_scene.spawnQuadCLT( + scenes[i] = ref_scene.spawnQuadCLTWithNoise( // spawnQuadCLT( sts[i], clt_parameters, colorProcParameters, // + noise_sigma_level, // double [] noise_sigma_level, threadsMax, -1); // debug_level); scenes[i].setDSRBG( @@ -3143,31 +3145,58 @@ public class OpticalFlow { -1); // debug_level); // int debugLevel) } - final double [][] combo_dsn = prepareInitialComboDS( clt_parameters, // final CLTParameters clt_parameters, scenes, // final QuadCLT [] scenes, indx_ref, // final int indx_ref, debug_level-2); // final int debug_level); - + final double [][] combo_dsn_change = new double [combo_dsn.length+1][]; + for (int i = 0; i < combo_dsn.length; i++) { + combo_dsn_change[i] = combo_dsn[i]; + } final int margin = 8; final int tilesX = ref_scene.getTileProcessor().getTilesX(); final int tilesY = ref_scene.getTileProcessor().getTilesY(); -//// final int tiles =tilesX * tilesY; - + + String [] combo_dsn_titles = {"disp", "strength", "num_valid","change"}; + if (debug_level >-3) { - String [] dbg_titles = {"disp", "strength", "num_valid"}; (new ShowDoubleFloatArrays()).showArrays( - combo_dsn, + combo_dsn_change, tilesX, tilesY, true, "combo_dsn-initial"+ref_scene.getImageName(), - dbg_titles); // dsrbg_titles); + combo_dsn_titles); // dsrbg_titles); } final int max_refines = 10; + final String [] iter_titles = {"disp", "diff", "strength"}; + final int [] iter_indices = {0,1,3}; + final int last_slices = combo_dsn_titles.length; + final int last_initial_slices = last_slices + iter_titles.length; + + final double [][] refine_results = new double [last_slices + 3 * (max_refines + 1)][]; + String [] refine_titles = new String [refine_results.length]; + for (int i = 0; i < combo_dsn_titles.length; i++) { + refine_results[i] = combo_dsn_change[i]; + refine_titles[i] = combo_dsn_titles[i]+"-last"; + } + for (int i = 0; i < iter_titles.length; i++) { + refine_titles[last_slices + i] = iter_titles[i]+"-initial"; + if (combo_dsn_change[iter_indices[i]] != null) { + refine_results[last_slices + i] = combo_dsn_change[iter_indices[i]].clone(); + } else { + refine_results[last_slices + i] = new double [tilesX * tilesY]; + } + } + for (int nrefine = 0; nrefine < max_refines; nrefine++) { + for (int i = 0; i < iter_titles.length; i++) { + refine_titles[last_initial_slices + i * max_refines + nrefine ] = combo_dsn_titles[i]+"-"+nrefine; + } + } + for (int nrefine = 0; nrefine < max_refines; nrefine++) { Runtime.getRuntime().gc(); System.out.println("--- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); @@ -3178,10 +3207,10 @@ public class OpticalFlow { clt_parameters, // final CLTParameters clt_parameters, scenes, // final QuadCLT [] scenes, indx_ref, // final int indx_ref, - combo_dsn[0], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid) + combo_dsn_change[0], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid) margin, // final int margin, nrefine, // final int nrefine, // just for debug title - debug_level); // final int debug_level) + debug_level-5); // final int debug_level) Runtime.getRuntime().gc(); System.out.println("--- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); @@ -3197,53 +3226,256 @@ public class OpticalFlow { // update disparities final int disparity_index = ImageDtt.DISPARITY_INDEX_CM; // 2 final int strength_index = ImageDtt.DISPARITY_STRENGTH_INDEX; // 10 - for (int nTile =0; nTile < combo_dsn[0].length; nTile++) { - if (!Double.isNaN(combo_dsn[0][nTile]) && !Double.isNaN(disparity_map[disparity_index][nTile])) { - combo_dsn[0][nTile] += disparity_map[disparity_index][nTile]; - combo_dsn[1][nTile] = disparity_map[strength_index][nTile]; + for (int nTile =0; nTile < combo_dsn_change[0].length; nTile++) { + if (!Double.isNaN(combo_dsn_change[0][nTile]) && !Double.isNaN(disparity_map[disparity_index][nTile])) { + combo_dsn_change[0][nTile] += disparity_map[disparity_index][nTile]; + combo_dsn_change[1][nTile] = disparity_map[strength_index][nTile]; } } + combo_dsn_change[combo_dsn_change.length -1] = disparity_map[disparity_index]; + + for (int i = 0; i < iter_titles.length; i++) { + refine_results[last_initial_slices + (i * max_refines) + nrefine] = combo_dsn_change[iter_indices[i]].clone(); + } + if (debug_level >-3) { - String [] dbg_titles = {"disp", "strength", "num_valid"}; (new ShowDoubleFloatArrays()).showArrays( - combo_dsn, + combo_dsn_change, tilesX, tilesY, true, "combo_dsn-"+nrefine+"-"+ref_scene.getImageName(), - dbg_titles); // dsrbg_titles); + combo_dsn_titles); // dsrbg_titles); } } -/* - if (debug_level > -10){ // -1 - ImageDtt image_dtt = new ImageDtt( - clt_parameters.transform_size, - ref_scene.isMonochrome(), - ref_scene.isLwir(), - clt_parameters.getScaleStrength(ref_scene.isAux()), - ref_scene.getGpuQuad()); - - double [][] dbg_corr_rslt_partial = image_dtt.corr_partial_dbg( - clt_corr_partial, - 2*8 - 1, //final int corr_size, - 4, // final int pairs, - 4, // final int colors, - clt_parameters.corr_border_contrast, + if (debug_level >-5) { + (new ShowDoubleFloatArrays()).showArrays( + refine_results, + tilesX, + tilesY, + true, + "combo-"+max_refines+"-"+ref_scene.getImageName(), + refine_titles); // dsrbg_titles); + } + ref_scene.saveDoubleArrayInModelDirectory( + "-results-nonoise", // String suffix, + refine_titles, // null, // String [] labels, // or null + refine_results, // dbg_data, // double [][] data, + tilesX, // int width, + tilesY); // int height) + + // save combo_dsn_change to model directory + if (debug_level >-100) { + return; + } + + System.out.println("IntersceneAccumulate(), got previous scenes: "+sts.length); + if (debug_level > 1) { // tested OK + System.out.println("IntersceneAccumulate(): preparing image set..."); + int nscenes = scenes.length; + // int indx_ref = nscenes - 1; + double [][][] all_scenes_xyzatr = new double [scenes.length][][]; // includes reference (last) + double [][][] all_scenes_ers_dt = new double [scenes.length][][]; // includes reference (last) + all_scenes_xyzatr[indx_ref] = new double [][] {ZERO3,ZERO3}; + all_scenes_ers_dt[indx_ref] = new double [][] { + ers_reference.getErsXYZ_dt(), + ers_reference.getErsATR_dt()}; + + for (int i = 0; i < nscenes; i++) if (i != indx_ref) { + String ts = scenes[i].getImageName(); + all_scenes_xyzatr[i] = new double[][] {ers_reference.getSceneXYZ(ts), ers_reference.getSceneATR(ts)}; + all_scenes_ers_dt[i] = new double[][] {ers_reference.getSceneErsXYZ_dt(ts), ers_reference.getSceneErsATR_dt(ts)}; + } + compareRefSceneTiles( + "" , // String suffix, + true, // false, // boolean blur_reference, + all_scenes_xyzatr, // double [][][] scene_xyzatr, // does not include reference + all_scenes_ers_dt, // double [][][] scene_ers_dt, // does not include reference + scenes, // QuadCLT [] scenes, + 8); // int iscale) // 8 + } + // create initial disparity map for the reference scene + + } + + public void IntersceneNoise( + CLTParameters clt_parameters, + boolean ref_only, // process only reference frame (false - inter-scene) + ColorProcParameters colorProcParameters, + QuadCLT ref_scene, // ordered by increasing timestamps + double [] noise_sigma_level, + int debug_level + ) + { + System.out.println("IntersceneNoise(), scene timestamp="+ref_scene.getImageName()); + ErsCorrection ers_reference = ref_scene.getErsCorrection(); + String [] sts = ref_only ? (new String [0]) : ers_reference.getScenes(); + // get list of all other scenes + int num_scenes = sts.length + 1; + int indx_ref = num_scenes - 1; + QuadCLT [] scenes = new QuadCLT [num_scenes]; + scenes[indx_ref] = ref_scene; + + for (int i = 0; i < sts.length; i++) { + scenes[i] = ref_scene.spawnQuadCLTWithNoise( // spawnQuadCLT( + sts[i], + clt_parameters, + colorProcParameters, // + noise_sigma_level, // double [] noise_sigma_level, threadsMax, - debug_level); - // titles.length = 15, corr_rslt_partial.length=16! - (new ShowDoubleFloatArrays()).showArrays( // out of boundary 15 - dbg_corr_rslt_partial, - tilesX*(2*8), - tilesY*(2*8), + -1); // debug_level); + scenes[i].setDSRBG( + clt_parameters, // CLTParameters clt_parameters, + threadsMax, // int threadsMax, // maximal number of threads to launch + updateStatus, // boolean updateStatus, + -1); // debug_level); // int debugLevel) + } + + double [][] combo_dsn = null; + if (noise_sigma_level == null) { + combo_dsn = prepareInitialComboDS( + clt_parameters, // final CLTParameters clt_parameters, + scenes, // final QuadCLT [] scenes, + indx_ref, // final int indx_ref, + debug_level-2); // final int debug_level); + } else { + combo_dsn = ref_scene. readDoubleArrayFromModelDirectory( + "-results-nonoise", // String suffix, + 3, // int num_slices, // (0 - all) + null); // int [] wh); + + } + final double [][] combo_dsn_change = new double [combo_dsn.length+1][]; + for (int i = 0; i < combo_dsn.length; i++) { + combo_dsn_change[i] = combo_dsn[i]; + } + final int margin = 8; + final int tilesX = ref_scene.getTileProcessor().getTilesX(); + final int tilesY = ref_scene.getTileProcessor().getTilesY(); + + String [] combo_dsn_titles = {"disp", "strength", "num_valid","change"}; + + if (debug_level > 0) { + (new ShowDoubleFloatArrays()).showArrays( + combo_dsn_change, + tilesX, + tilesY, true, - ref_scene.getImageName()+"-TD-PART_CORR-D"+clt_parameters.disparity, - ImageDtt.CORR_TITLES); - // titles); + "combo_dsn-initial"+ref_scene.getImageName(), + combo_dsn_titles); // dsrbg_titles); } -*/ + final int max_refines = 10; + final String [] iter_titles = {"disp", "diff", "strength"}; + final int [] iter_indices = {0,1,3}; + final int last_slices = combo_dsn_titles.length; + final int last_initial_slices = last_slices + iter_titles.length; + + final double [][] refine_results = new double [last_slices + 3 * (max_refines + 1)][]; + String [] refine_titles = new String [refine_results.length]; + for (int i = 0; i < combo_dsn_titles.length; i++) { + refine_results[i] = combo_dsn_change[i]; + refine_titles[i] = combo_dsn_titles[i]+"-last"; + } + for (int i = 0; i < iter_titles.length; i++) { + refine_titles[last_slices + i] = iter_titles[i]+"-initial"; + if (combo_dsn_change[iter_indices[i]] != null) { + refine_results[last_slices + i] = combo_dsn_change[iter_indices[i]].clone(); + } else { + refine_results[last_slices + i] = new double [tilesX * tilesY]; + } + } + for (int nrefine = 0; nrefine < max_refines; nrefine++) { + for (int i = 0; i < iter_titles.length; i++) { + refine_titles[last_initial_slices + i * max_refines + nrefine ] = combo_dsn_titles[iter_indices[i]]+"-"+nrefine; + } + } + if (noise_sigma_level != null) { // add initial offset to the expected disparity + for (int i = 0; i < combo_dsn_change[0].length; i++) { + combo_dsn_change[0][i] += noise_sigma_level[2]; + } + } + for (int nrefine = 0; nrefine < max_refines; nrefine++) { + Runtime.getRuntime().gc(); + System.out.println("--- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); + + double [][] disparity_map = + // double [][][][][] clt_corr_partial = + correlateInterscene( + clt_parameters, // final CLTParameters clt_parameters, + scenes, // final QuadCLT [] scenes, + indx_ref, // final int indx_ref, + combo_dsn_change[0], // final double [] disparity_ref, // disparity in the reference view tiles (Double.NaN - invalid) + margin, // final int margin, + nrefine, // final int nrefine, // just for debug title + debug_level-5); // final int debug_level) + + Runtime.getRuntime().gc(); + System.out.println("--- Free memory="+Runtime.getRuntime().freeMemory()+" (of "+Runtime.getRuntime().totalMemory()+")"); + if (debug_level >0) { + (new ShowDoubleFloatArrays()).showArrays( + disparity_map, + tilesX, + tilesY, + true, + "accumulated_disparity_map-"+nrefine, + ImageDtt.DISPARITY_TITLES + ); + } + // update disparities + final int disparity_index = ImageDtt.DISPARITY_INDEX_CM; // 2 + final int strength_index = ImageDtt.DISPARITY_STRENGTH_INDEX; // 10 + for (int nTile =0; nTile < combo_dsn_change[0].length; nTile++) { + if (!Double.isNaN(combo_dsn_change[0][nTile]) && !Double.isNaN(disparity_map[disparity_index][nTile])) { + combo_dsn_change[0][nTile] += disparity_map[disparity_index][nTile]; + combo_dsn_change[1][nTile] = disparity_map[strength_index][nTile]; + } + } + combo_dsn_change[combo_dsn_change.length -1] = disparity_map[disparity_index]; + + for (int i = 0; i < iter_titles.length; i++) { + refine_results[last_initial_slices + (i * max_refines) + nrefine] = combo_dsn_change[iter_indices[i]].clone(); + } + + if (debug_level >0) { + (new ShowDoubleFloatArrays()).showArrays( + combo_dsn_change, + tilesX, + tilesY, + true, + "combo_dsn-"+nrefine+"-"+ref_scene.getImageName(), + combo_dsn_titles); // dsrbg_titles); + } + } + + if (debug_level > 0) { + (new ShowDoubleFloatArrays()).showArrays( + refine_results, + tilesX, + tilesY, + true, + "combo-"+max_refines+"-"+ref_scene.getImageName(), + refine_titles); // dsrbg_titles); + } + //noise_sigma_level + String rslt_suffix = "-results-nonoise"; + if (noise_sigma_level != null) { + rslt_suffix = "-results-lev_"+noise_sigma_level[0]+"-sigma_"+noise_sigma_level[1]+"-offset"+noise_sigma_level[2]; + if (ref_only) { + rslt_suffix +="-nointer"; + } else { + rslt_suffix +="-inter"; + } + rslt_suffix +="-mask"+clt_parameters.img_dtt.dbg_pair_mask; + } + ref_scene.saveDoubleArrayInModelDirectory( + rslt_suffix, // String suffix, + refine_titles, // null, // String [] labels, // or null + refine_results, // dbg_data, // double [][] data, + tilesX, // int width, + tilesY); // int height) + // save combo_dsn_change to model directory if (debug_level >-100) { return; } @@ -3278,6 +3510,8 @@ public class OpticalFlow { } + + public double [][] prepareInitialComboDS( final CLTParameters clt_parameters, final QuadCLT [] scenes, @@ -3401,7 +3635,6 @@ public class OpticalFlow { public double[][] correlateInterscene( -// public double [][][][][] correlateInterscene( final CLTParameters clt_parameters, final QuadCLT [] scenes, final int indx_ref, @@ -3411,6 +3644,7 @@ public class OpticalFlow { final int debug_level ) { + // Debug images if debug_level > -2 final double fat_zero_pre = (debug_level>-100)?-1.0:0.0; //100000.0; // 10000.0; // 1000.0; // final double output_amplitude = 30000; // 10000; // 1000; // 200; // 50.0; final int num_scenes = scenes.length; @@ -3426,16 +3660,7 @@ public class OpticalFlow { ref_scene.isMonochrome(), ref_scene.isLwir(), clt_parameters.getScaleStrength(ref_scene.isAux()), - ref_scene.getGpuQuad()); - /* - double [][][][][] clt_corr_partial = null; - clt_corr_partial = new double [tilesY][tilesX][][][]; - for (int i = 0; i < tilesY; i++){ - for (int j = 0; j < tilesX; j++){ - clt_corr_partial[i][j] = null; - } - } - */ + ref_scene.getGPU()); double [][] disparity_map = new double [ImageDtt.DISPARITY_TITLES.length][]; int disparity_modes = @@ -3446,8 +3671,8 @@ public class OpticalFlow { // final Rectangle tile_woi = new Rectangle(81,77,64,97); // for visualizations final Rectangle tile_woi = new Rectangle(30,70,50,110); // for visualizations final int vis_gap = 2; - final float [][] vis_corr_td = new float[num_scenes + 1][]; // transform-domain visualization - final float [][] vis_corr_pd = new float[num_scenes + 2][]; // pixel-domain visualization + final float [][] vis_corr_td = (debug_level > -2) ? (new float[num_scenes + 1][]) : null; // transform-domain visualization + final float [][] vis_corr_pd = (debug_level > -2) ? (new float[num_scenes + 2][]) : null; // pixel-domain visualization final int [] wis_wh = new int [2]; final float [][][][] fclt_corrs = new float [num_scenes+1][tilesX*tilesY][][]; // will only contain tile_woi tiles to save memory for (int i = 0; i < num_scenes; i++) { @@ -3483,8 +3708,8 @@ public class OpticalFlow { scenes[i], // final QuadCLT scene_QuadClt, ref_scene); // final QuadCLT reference_QuadClt) } - if (scenes[i].getGpuQuad().getQuadCLT() != scenes[i]) { - scenes[i].getGpuQuad().updateQuadCLT(scenes[i]); // to re-load new set of Bayer images to the GPU + if (scenes[i].getGPU().getQuadCLT() != scenes[i]) { + scenes[i].getGPU().updateQuadCLT(scenes[i]); // to re-load new set of Bayer images to the GPU } final double disparity_corr = 0.0; // (z_correction == 0) ? 0.0 : geometryCorrection.getDisparityFromZ(1.0/z_correction); final double gpu_sigma_corr = clt_parameters.getGpuCorrSigma(scenes[i].isMonochrome()); @@ -3510,6 +3735,8 @@ public class OpticalFlow { clt_parameters.corr_blue, // +used threadsMax, // final int threadsMax, // maximal number of threads to launch debug_level); // final int globalDebugLevel) + + // experimental, not currently uxsed if (fat_zero_pre >= 0.0) { ImageDtt.corr_td_normalize( fcorrs_td[i], // final float [][][][] fcorr_td, // will be updated @@ -3662,7 +3889,7 @@ public class OpticalFlow { ImageDtt.startAndJoin(threads); // // TODO: Visualize each correlation - individual and combo // testing: overwrite with reference frame correlations - if (debug_level < -100) { + if (debug_level < -100) { for (int i = 0; i < fcorr_combo_td.length; i++) { fcorr_combo_td[i] = fcorrs_combo_td[indx_ref][i].clone(); } @@ -3674,8 +3901,9 @@ public class OpticalFlow { fcorrs_td[i] = null; fcorrs_combo_td[i] = null; } - if (debug_level > -10){ // -1 - final int indx_corr = -1; + if (debug_level > -2){ // -1 + int indx_corr = -1; + indx_corr = -1; final float [][][][] fcorr_td_dbg = (indx_corr < 0) ? fcorr_td : fcorrs_td[indx_corr]; final float [][][][] fcorr_combo_td_dbg = (indx_corr < 0) ? fcorr_combo_td : fcorrs_combo_td[indx_corr]; int [] wh = new int[2]; @@ -3751,17 +3979,6 @@ public class OpticalFlow { clt_parameters.tileY, // final int debug_tileY, threadsMax, debug_level -1 ); - /* - (new ShowDoubleFloatArrays()).showArrays( - disparity_map, - tilesX, - tilesY, - true, - "accumulated_disparity_map", - ImageDtt.DISPARITY_TITLES -// ,dbg_titles - ); - */ if (vis_corr_pd != null) { // add combined data as the last slice vis_corr_pd[num_scenes] = ImageDtt.corr_partial_wnd( // not used in lwir fclt_corr, // clt_corr_partial, // final double [][][][][] corr_data, @@ -3789,13 +4006,12 @@ public class OpticalFlow { - if (debug_level > -10){ // -1 + if (debug_level > -2){ // -1 float [][] dbg_corr_rslt_partial = ImageDtt.corr_partial_dbg( fclt_corr, // final float [][][] fcorr_data, // [tile][index][(2*transform_size-1)*(2*transform_size-1)] // if null - will not calculate tilesX, //final int tilesX, 2*image_dtt.transform_size - 1, //final int corr_size, 10, // final int layers 4, // final int pairs, -// 4, // final int colors, clt_parameters.corr_border_contrast, threadsMax, debug_level); @@ -3865,10 +4081,7 @@ public class OpticalFlow { "PD-"+"FZ-"+(clt_parameters.getGpuFatZero(ref_scene.isMonochrome()))+"-"+tile_woi.x+"_"+tile_woi.y+"-"+nrefine, dbg_titles); } - return disparity_map; // disparity_map -// return clt_corr_partial; // disparity_map - } @@ -3969,7 +4182,7 @@ public class OpticalFlow { }; } ImageDtt.startAndJoin(threads); - scene.getGpuQuad().setInterTasks( + scene.getGPU().setInterTasks( pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls) scene.getGeometryCorrection(), // final GeometryCorrection geometryCorrection, disparity_corr, // final double disparity_corr, diff --git a/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java b/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java index 0781a98..150443e 100644 --- a/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java +++ b/src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java @@ -37,6 +37,7 @@ import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.util.Arrays; import java.util.Properties; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.DoubleAccumulator; @@ -51,9 +52,19 @@ import com.elphel.imagej.gpu.GPUTileProcessor; import ij.ImagePlus; import ij.ImageStack; +import ij.Prefs; +import ij.io.FileSaver; public class QuadCLT extends QuadCLTCPU { int dbg_lev = 1; - private GPUTileProcessor.GpuQuad gpuQuad = null; + private GPUTileProcessor.GpuQuad gpuQuad = null; // use updateQuadCLT() to update after switching to a different + // QuadCLT instance, this.getGPU().getQuadCLT() should be equal to this + /* + if (getGPU().getQuadCLT() != this) { + getGPU().updateQuadCLT(this); // to re-load new set of Bayer images to the GPU + } + + */ + public QuadCLT( String prefix, Properties properties, @@ -74,14 +85,12 @@ public class QuadCLT extends QuadCLTCPU { } } - public GPUTileProcessor.GpuQuad getGpuQuad(){ - return this.gpuQuad; - } public QuadCLT restoreFromModel( - CLTParameters clt_parameters, - ColorProcParameters colorProcParameters, // - int threadsMax, - int debugLevel) + CLTParameters clt_parameters, + ColorProcParameters colorProcParameters, + double [] noise_sigma_level, + int threadsMax, + int debugLevel) { final int debugLevelInner=clt_parameters.batch_run? -2: debugLevel; @@ -117,6 +126,13 @@ public class QuadCLT extends QuadCLTCPU { saturation_imp, // output // boolean [][] saturation_imp, threadsMax, // int threadsMax, debugLevelInner); // int debugLevel); + if (noise_sigma_level != null) { + generateAddNoise( + "-NOISE", + noise_sigma_level, + threadsMax, + 1); // debugLevel); // final int debug_level) + } restoreDSI("-DSI_MAIN"); // "-DSI_COMBO", "-DSI_MAIN" (DSI_COMBO_SUFFIX, DSI_MAIN_SUFFIX) restoreInterProperties( // restore properties for interscene processing (extrinsics, ers, ...) null, // String path, // full name with extension or null to use x3d directory @@ -125,7 +141,210 @@ public class QuadCLT extends QuadCLTCPU { // showDSIMain(); return this; // can only be QuadCLT instance } + + // generate and save noise file (each Bayer component amplitude same as the corresponding image average, + // apply gaussian blur with sigma (before Bayer scaling) + // If file with the same sigma already exists in the model directory - just use it, multiply by noise_sigma_level[0] and add to the non-zero Bayer + + public void generateAddNoise( + final String suffix, + final double [] noise_sigma_level, + final int threadsMax, + final int debug_level) + { + final double scale =noise_sigma_level[0]; + final double sigma =noise_sigma_level[1]; + final int num_cams = this.image_data.length; + final int num_cols = image_data[0].length; + final int [] image_wh = geometryCorrection.getSensorWH(); + String x3d_path= correctionsParameters.selectX3dDirectory( // for x3d and obj + correctionsParameters.getModelName(image_name), // quad timestamp. Will be ignored if correctionsParameters.use_x3d_subdirs is false + correctionsParameters.x3dModelVersion, + true, // smart, + true); //newAllowed, // save + String noise_suffix = suffix + sigma; + String file_name = image_name + noise_suffix; + String file_path = x3d_path + Prefs.getFileSeparator() + file_name + ".tiff"; + ImagePlus imp = null; + try { + imp = new ImagePlus(file_path); + } catch (Exception e) { + System.out.println ("Failed to open "+file_path+", will generate it"); + } + final Thread[] threads = ImageDtt.newThreadArray(threadsMax); + final AtomicInteger ai = new AtomicInteger(0); + if ((imp == null) || (imp.getTitle() == null) || (imp.getTitle().equals(""))) { + System.out.println ("Empty "+file_path+", will generate it"); + int num_pix = image_wh[0] * image_wh[1]; + final double [][] noise = new double [num_cams][num_pix]; + for (int q = 0; q < num_cams; q++) { + final int fq = q; + ai.set(0); + for (int ithread = 0; ithread < threads.length; ithread++) { + threads[ithread] = new Thread() { + public void run() { + Random random = new Random(); + for (int i = ai.getAndIncrement(); i < noise[0].length; i = ai.getAndIncrement()) { + noise[fq][i] = random.nextGaussian(); + } + } + }; + } + ImageDtt.startAndJoin(threads); + } + ai.set(0); + if (sigma > 0) { + for (int ithread = 0; ithread < threads.length; ithread++) { + threads[ithread] = new Thread() { + public void run() { + for (int q = ai.getAndIncrement(); q 100) { + double [][] dbg_data = new double [num_cams*num_cols][]; + for (int q = 0; q < num_cams;q++) { + for (int c = 0; c < num_cols; c++) { + dbg_data[q*num_cols+c] = image_data[q][c]; + } + } + saveDoubleArrayInModelDirectory( + noise_suffix + "-MIXED"+noise_sigma_level[0], // String suffix, + null, // String [] labels, // or null + dbg_data, // double [][] data, + image_wh[0], // int width, + image_wh[1]); // int height) + } + } + + public ImagePlus saveDoubleArrayInModelDirectory( + String suffix, + String [] labels, // or null + double [][] data, + int width, + int height) + { + String x3d_path= correctionsParameters.selectX3dDirectory( // for x3d and obj + correctionsParameters.getModelName(image_name), // quad timestamp. Will be ignored if correctionsParameters.use_x3d_subdirs is false + correctionsParameters.x3dModelVersion, + true, // smart, + true); //newAllowed, // save + String file_name = image_name + suffix; + String file_path = x3d_path + Prefs.getFileSeparator() + file_name + ".tiff"; + ImageStack imageStack = (new ShowDoubleFloatArrays()).makeStack(data, width, height, labels); + ImagePlus imp = new ImagePlus( file_name, imageStack); + FileSaver fs=new FileSaver(imp); + fs.saveAsTiff(file_path); + return imp; + } + + public double [][] readDoubleArrayFromModelDirectory( + String suffix, + int num_slices, // (0 - all) + int [] wh + ) + { +// final int [] image_wh = geometryCorrection.getSensorWH(); + String x3d_path= correctionsParameters.selectX3dDirectory( // for x3d and obj + correctionsParameters.getModelName(image_name), // quad timestamp. Will be ignored if correctionsParameters.use_x3d_subdirs is false + correctionsParameters.x3dModelVersion, + true, // smart, + true); //newAllowed, // save + String file_name = image_name + suffix; + String file_path = x3d_path + Prefs.getFileSeparator() + file_name + ".tiff"; + ImagePlus imp = null; + try { + imp = new ImagePlus(file_path); + } catch (Exception e) { + System.out.println ("Failed to open "+file_path+", will generate it"); + } + if ((imp == null) || (imp.getTitle() == null) || (imp.getTitle().equals(""))) { + return null; + } + ImageStack imageStack = imp.getStack(); + int nChn=imageStack.getSize(); + if ((num_slices > 0) && (num_slices < nChn)) { + nChn = num_slices; + } + float [] fpixels; + double [][] result = new double [nChn][]; + for (int n = 0; n < nChn; n++) { + fpixels = (float[]) imageStack.getPixels(n + 1); + result[n] = new double [fpixels.length]; + for (int i = 0; i < fpixels.length; i++) { + result[n][i] = fpixels[i]; + } + } + if (wh != null) { + wh[0] = imp.getWidth(); + wh[1] = imp.getHeight(); + } + return result; + } + + + public static double [] removeDisparityOutliers( final double [][] ds0, final double max_strength, // do not touch stronger @@ -1744,8 +1963,47 @@ public class QuadCLT extends QuadCLTCPU { public GPUTileProcessor.GpuQuad getGPU() { return this.gpuQuad; } - public void processCLTQuadCorrGPU( - ImagePlus [] imp_quad, + + + public void genSave4sliceImage( + CLTParameters clt_parameters, + String suffix, + EyesisCorrectionParameters.DebayerParameters debayerParameters, + ColorProcParameters colorProcParameters, + CorrectionColorProc.ColorGainsParameters channelGainParameters, + EyesisCorrectionParameters.RGBParameters rgbParameters, + final int threadsMax, // maximal number of threads to launch + final int debugLevel){ + String x3d_path= correctionsParameters.selectX3dDirectory( // for x3d and obj + correctionsParameters.getModelName(image_name), // quad timestamp. Will be ignored if correctionsParameters.use_x3d_subdirs is false + correctionsParameters.x3dModelVersion, + true, // smart, + true); //newAllowed, // save + String file_name = image_name + suffix; + String file_path = x3d_path + Prefs.getFileSeparator() + file_name + ".tiff"; + if (getGPU().getQuadCLT() != this) { + getGPU().updateQuadCLT(this); // to re-load new set of Bayer images to the GPU + } + + ImagePlus img_noise = processCLTQuadCorrGPU( + null, // ImagePlus [] imp_quad, //null will be OK + null, // boolean [][] saturation_imp, // (near) saturated pixels or null // Not needed use this.saturation_imp + clt_parameters, // CLTParameters clt_parameters, + debayerParameters, // EyesisCorrectionParameters.DebayerParameters debayerParameters, + colorProcParameters, // ColorProcParameters colorProcParameters, + channelGainParameters, // CorrectionColorProc.ColorGainsParameters channelGainParameters, + rgbParameters, // EyesisCorrectionParameters.RGBParameters rgbParameters, + null, // double [] scaleExposures, // probably not needed here - restores brightness of the final image + true, // boolean only4slice, + threadsMax, // final int threadsMax, // maximal number of threads to launch + false, // final boolean updateStatus, + debugLevel); // final int debugLevel); + FileSaver fs=new FileSaver(img_noise); + fs.saveAsTiff(file_path); + } + + public ImagePlus processCLTQuadCorrGPU( + ImagePlus [] imp_quad, //null will be OK boolean [][] saturation_imp, // (near) saturated pixels or null // Not needed use this.saturation_imp CLTParameters clt_parameters, EyesisCorrectionParameters.DebayerParameters debayerParameters, @@ -1753,6 +2011,7 @@ public class QuadCLT extends QuadCLTCPU { CorrectionColorProc.ColorGainsParameters channelGainParameters, EyesisCorrectionParameters.RGBParameters rgbParameters, double [] scaleExposures, // probably not needed here - restores brightness of the final image + boolean only4slice, final int threadsMax, // maximal number of threads to launch final boolean updateStatus, final int debugLevel){ @@ -1773,7 +2032,7 @@ public class QuadCLT extends QuadCLTCPU { updateStatus, // final boolean updateStatus, debugLevel); // final int debugLevel) - return; + return null; } // GPU-specific @@ -1781,18 +2040,6 @@ public class QuadCLT extends QuadCLTCPU { boolean is_lwir = isLwir(); final boolean batch_mode = clt_parameters.batch_run; //disable any debug images -/* - double fat_zero = clt_parameters.getGpuFatZero(is_mono); // 30.0; - double [] scales = (is_mono) ? (new double [] {1.0}) :(new double [] { - clt_parameters.gpu_weight_r, // 0.25 - clt_parameters.gpu_weight_b, // 0.25 - 1.0 - clt_parameters.gpu_weight_r - clt_parameters.gpu_weight_b}); // 0.5 - double cwgreen = 1.0/(1.0 + clt_parameters.corr_red + clt_parameters.corr_blue); // green color - double [] col_weights= (is_mono) ? (new double [] {1.0}) :(new double [] { - clt_parameters.corr_red * cwgreen, - clt_parameters.corr_blue * cwgreen, - cwgreen}); -*/ ImageDtt image_dtt = new ImageDtt( clt_parameters.transform_size, is_mono, @@ -1834,10 +2081,13 @@ public class QuadCLT extends QuadCLTCPU { boolean advanced= correctionsParameters.zcorrect || correctionsParameters.equirectangular; boolean toRGB= advanced? true: correctionsParameters.toRGB; - ImagePlus [] results = new ImagePlus[imp_quad.length]; - for (int i = 0; i < results.length; i++) { - results[i] = imp_quad[i]; - results[i].setTitle(results[i].getTitle()+"RAW"); + ImagePlus [] results = null; + if (imp_quad != null) { + results = new ImagePlus[imp_quad.length]; + for (int i = 0; i < results.length; i++) { + results[i] = imp_quad[i]; + results[i].setTitle(results[i].getTitle()+"RAW"); + } } if (debugLevel>1) System.out.println("processing: "+gpuQuad.quadCLT); @@ -1857,7 +2107,7 @@ public class QuadCLT extends QuadCLTCPU { debugLevel); // final int debugLevel) - not yet used if (tp_tasks.length == 0) { System.out.println("Empty tasks - nothing to do"); - return; + return null; } gpuQuad.setTasks( // copy tp_tasks to the GPU memory @@ -1877,15 +2127,11 @@ public class QuadCLT extends QuadCLTCPU { int out_width = gpuQuad.getImageWidth() + gpuQuad.getDttSize(); int out_height = gpuQuad.getImageHeight() + gpuQuad.getDttSize(); -// int tilesX = gpuQuad.getImageWidth() / gpuQuad.getDttSize(); -// int tilesY = gpuQuad.getImageHeight() / gpuQuad.getDttSize(); /* Prepare 4-channel images*/ ImagePlus [] imps_RGB = new ImagePlus[iclt_fimg.length]; for (int ncam = 0; ncam < iclt_fimg.length; ncam++) { -// String title=image_name+"-"+String.format("%02d", ncam); String title=String.format("%s%s-%02d",image_name, sAux(), ncam); - imps_RGB[ncam] = linearStackToColor( // probably no need to separate and process the second half with quadCLT_aux clt_parameters, colorProcParameters, @@ -1903,11 +2149,11 @@ public class QuadCLT extends QuadCLTCPU { debugLevel ); } - if (clt_parameters.gen_chn_img) { // save and show 4-slice image + if (clt_parameters.gen_chn_img || only4slice) { // save and show 4-slice image // combine to a sliced color image // assuming total number of images to be multiple of 4 // int [] slice_seq = {0,1,3,2}; //clockwise - int [] slice_seq = new int[results.length]; + int [] slice_seq = new int[gpuQuad.getNumCams()]; //results.length]; for (int i = 0; i < slice_seq.length; i++) { slice_seq[i] = i ^ ((i >> 1) & 1); // 0,1,3,2,4,5,7,6, ... } @@ -1915,22 +2161,26 @@ public class QuadCLT extends QuadCLTCPU { int height = imps_RGB[0].getHeight(); ImageStack array_stack=new ImageStack(width,height); for (int i = 0; i -100) return true; // temporarily ! public void intersceneAccumulate( QuadCLT quadCLT_main, // tiles should be set - CLTParameters clt_parameters, + CLTParameters clt_parameters, EyesisCorrectionParameters.DebayerParameters debayerParameters, ColorProcParameters colorProcParameters, CorrectionColorProc.ColorGainsParameters channelGainParameters, @@ -8559,6 +8561,7 @@ if (debugLevel > -100) return true; // temporarily ! final boolean updateStatus, final int debugLevel) throws Exception { +// double [] noise_sigma_level = {0.01, 1.5}; if ((quadCLT_main != null) && (quadCLT_main.getGPU() != null)) { quadCLT_main.getGPU().resetGeometryCorrection(); quadCLT_main.gpuResetCorrVector(); // .getGPU().resetGeometryCorrectionVector(); @@ -8573,10 +8576,11 @@ if (debugLevel > -100) return true; // temporarily ! } QuadCLT.SetChannels [] set_channels=quadCLT_main.setChannels(debugLevel); // TODO: use just the last one (to need this is no time) - QuadCLT ref_quadCLT = quadCLT_main.spawnQuadCLT( + QuadCLT ref_quadCLT = quadCLT_main.spawnQuadCLTWithNoise( // spawnQuadCLT( set_channels[set_channels.length-1].set_name, clt_parameters, colorProcParameters, // + null, // noise_sigma_level, // double [] noise_sigma_level, threadsMax, debugLevel); // temporarily fix wrong sign: @@ -8595,10 +8599,441 @@ if (debugLevel > -100) return true; // temporarily ! clt_parameters, // CLTParameters clt_parameters, colorProcParameters, // ColorProcParameters colorProcParameters, ref_quadCLT, // QuadCLT [] scenes, // ordered by increasing timestamps + null, // noise_sigma_level, // double [] noise_sigma_level, clt_parameters.ofp.debug_level_optical); // 1); // -1); // int debug_level); System.out.println("End of intersceneAccumulate()"); } + + public void intersceneNoiseStats( + QuadCLT quadCLT_main, // tiles should be set + CLTParameters clt_parameters, + EyesisCorrectionParameters.DebayerParameters debayerParameters, + ColorProcParameters colorProcParameters, + CorrectionColorProc.ColorGainsParameters channelGainParameters, + EyesisCorrectionParameters.RGBParameters rgbParameters, + EyesisCorrectionParameters.EquirectangularParameters equirectangularParameters, + Properties properties, + final int threadsMax, // maximal number of threads to launch + final boolean updateStatus, + final int debugLevel) throws Exception + { +// double [] noise_sigma_level = {0.01, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {0.1, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {1.0, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {3.0, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {5.0, 1.5, 1.0}; // amount, sigma, offset + double [] noise_sigma_level = null; + if (clt_parameters.inp.noise_scale > 0.0) { + noise_sigma_level = new double[] { + clt_parameters.inp.noise_scale, + clt_parameters.inp.noise_sigma, + clt_parameters.inp.initial_offset}; // amount, sigma, offset + } + + boolean ref_only = clt_parameters.inp.ref_only; // true; // process only reference frame (false - inter-scene) + if ((quadCLT_main != null) && (quadCLT_main.getGPU() != null)) { + quadCLT_main.getGPU().resetGeometryCorrection(); + quadCLT_main.gpuResetCorrVector(); // .getGPU().resetGeometryCorrectionVector(); + } + // final boolean batch_mode = clt_parameters.batch_run; + this.startTime=System.nanoTime(); + String [] sourceFiles0=quadCLT_main.correctionsParameters.getSourcePaths(); + QuadCLT.SetChannels [] set_channels_main = quadCLT_main.setChannels(debugLevel); + if ((set_channels_main == null) || (set_channels_main.length==0)) { + System.out.println("No files to process (of "+sourceFiles0.length+")"); + return; + } + QuadCLT.SetChannels [] set_channels=quadCLT_main.setChannels(debugLevel); // TODO: use just the last one (to need this is no time) + + QuadCLT ref_quadCLT = quadCLT_main.spawnQuadCLTWithNoise( // spawnQuadCLT( + set_channels[set_channels.length-1].set_name, + clt_parameters, + colorProcParameters, // + noise_sigma_level, // double [] noise_sigma_level, + threadsMax, + clt_parameters.inp.noise_debug_level); // debugLevel); + /**/ + getNoiseStats( + clt_parameters, // CLTParameters clt_parameters, + ref_quadCLT, //QuadCLT ref_scene, // ordered by increasing timestamps + debugLevel); // int debug_level); + /**/ + } + + + public void intersceneNoise( + QuadCLT quadCLT_main, // tiles should be set + CLTParameters clt_parameters, + EyesisCorrectionParameters.DebayerParameters debayerParameters, + ColorProcParameters colorProcParameters, + CorrectionColorProc.ColorGainsParameters channelGainParameters, + EyesisCorrectionParameters.RGBParameters rgbParameters, + EyesisCorrectionParameters.EquirectangularParameters equirectangularParameters, + Properties properties, + final int threadsMax, // maximal number of threads to launch + final boolean updateStatus, + final int debugLevel) throws Exception + { +// double [] noise_sigma_level = {0.01, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {0.1, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {1.0, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {3.0, 1.5, 1.0}; // amount, sigma, offset +// double [] noise_sigma_level = {5.0, 1.5, 1.0}; // amount, sigma, offset + double [] noise_sigma_level = null; + if (clt_parameters.inp.noise_scale > 0.0) { + noise_sigma_level = new double[] { + clt_parameters.inp.noise_scale, + clt_parameters.inp.noise_sigma, + clt_parameters.inp.initial_offset}; // amount, sigma, offset + } + + boolean ref_only = clt_parameters.inp.ref_only; // true; // process only reference frame (false - inter-scene) + if ((quadCLT_main != null) && (quadCLT_main.getGPU() != null)) { + quadCLT_main.getGPU().resetGeometryCorrection(); + quadCLT_main.gpuResetCorrVector(); // .getGPU().resetGeometryCorrectionVector(); + } + // final boolean batch_mode = clt_parameters.batch_run; + this.startTime=System.nanoTime(); + String [] sourceFiles0=quadCLT_main.correctionsParameters.getSourcePaths(); + QuadCLT.SetChannels [] set_channels_main = quadCLT_main.setChannels(debugLevel); + if ((set_channels_main == null) || (set_channels_main.length==0)) { + System.out.println("No files to process (of "+sourceFiles0.length+")"); + return; + } + QuadCLT.SetChannels [] set_channels=quadCLT_main.setChannels(debugLevel); // TODO: use just the last one (to need this is no time) + + QuadCLT ref_quadCLT = quadCLT_main.spawnQuadCLTWithNoise( // spawnQuadCLT( + set_channels[set_channels.length-1].set_name, + clt_parameters, + colorProcParameters, // + noise_sigma_level, // double [] noise_sigma_level, + threadsMax, + clt_parameters.inp.noise_debug_level); // debugLevel); + /* + getNoiseStats( + clt_parameters, // CLTParameters clt_parameters, + ref_quadCLT, //QuadCLT ref_scene, // ordered by increasing timestamps + debugLevel); // int debug_level); + */ + // Create 4-slice image with noise from the current data + if (noise_sigma_level != null) { + String noisy_4slice_suffix = "-noise-level_"+ noise_sigma_level[0]+"-sigma_"+noise_sigma_level[1]; + ref_quadCLT.genSave4sliceImage( + clt_parameters, // CLTParameters clt_parameters, + noisy_4slice_suffix, // String suffix, + debayerParameters, // EyesisCorrectionParameters.DebayerParameters debayerParameters, + colorProcParameters, // ColorProcParameters colorProcParameters, + channelGainParameters, // CorrectionColorProc.ColorGainsParameters channelGainParameters, + rgbParameters, // EyesisCorrectionParameters.RGBParameters rgbParameters, + threadsMax, // final int threadsMax, // maximal number of threads to launch + clt_parameters.inp.noise_debug_level); // debugLevel); // final int debugLevel); + } + // temporarily fix wrong sign: +// ErsCorrection ers = (ErsCorrection) (ref_quadCLT.getGeometryCorrection()); + ref_quadCLT.setDSRBG( // runs GPU to calculate average R,B,G + clt_parameters, // CLTParameters clt_parameters, + threadsMax, // int threadsMax, // maximal number of threads to launch + updateStatus, // boolean updateStatus, + clt_parameters.inp.noise_debug_level); // debugLevel); // int debugLevel) + +// if (debugLevel > -1000) return; // TODO: Remove + + + OpticalFlow opticalFlow = new OpticalFlow( + threadsMax, // int threadsMax, // maximal number of threads to launch + updateStatus); // boolean updateStatus); + + opticalFlow.IntersceneNoise( + clt_parameters, // CLTParameters clt_parameters, + ref_only, // boolean ref_only, // process only reference frame (false - inter-scene) + colorProcParameters, // ColorProcParameters colorProcParameters, + ref_quadCLT, // QuadCLT [] scenes, // ordered by increasing timestamps + noise_sigma_level, // double [] noise_sigma_level, + clt_parameters.inp.noise_debug_level); // clt_parameters.ofp.debug_level_optical - 1); // 1); // -1); // int debug_level); + System.out.println("End of intersceneNoise()"); + } + + public void getNoiseStats( + CLTParameters clt_parameters, + QuadCLT ref_scene, // ordered by increasing timestamps + int debug_level) + { + String [] noise_files = { + "-results-lev_1.0E-6-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_1.0E-6-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_1.0E-6-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.01-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.01-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.01-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.1-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.1-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.1-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.2-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.2-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.2-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.3-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.3-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.3-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.4-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.4-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.4-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.5-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.5-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.5-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.6-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.6-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.6-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.7-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.7-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.7-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.8-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.8-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.8-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_0.9-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_0.9-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_0.9-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_1.0-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_1.0-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_1.0-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_1.2-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_1.2-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_1.2-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_1.4-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_1.4-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_1.4-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_1.6-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_1.6-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_1.6-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_1.8-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_1.8-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_1.8-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_2.0-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_2.0-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_2.0-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_2.2-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_2.2-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_2.2-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_2.4-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_2.4-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_2.4-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_2.6-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_2.6-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_2.6-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_2.8-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_2.8-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_2.8-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_3.0-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_3.0-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_3.0-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_3.3-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_3.3-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_3.3-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_3.6-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_3.6-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_3.6-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_4.0-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_4.0-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_4.0-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_4.5-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_4.5-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_4.5-sigma_1.5-offset1.0-nointer-mask1", + + "-results-lev_5.0-sigma_1.5-offset1.0-inter-mask63", + "-results-lev_5.0-sigma_1.5-offset1.0-nointer-mask63", + "-results-lev_5.0-sigma_1.5-offset1.0-nointer-mask1"}; + + getNoiseStats( + clt_parameters, + ref_scene, // ordered by increasing timestamps + noise_files, + debug_level); + } + + + + + public void getNoiseStats( + CLTParameters clt_parameters, + QuadCLT ref_scene, // ordered by increasing timestamps + String [] noise_files, + int debug_level) + { + /* + "disp-last", + "str_last", + "num vlaid" <= 1.0 + */ + double max_diff = 0.01; // last diff > + double max_err = 2.0; // 1.0; // 0.5; // pix + double max_err1 =0.25; // pix + double min_strength = 0.0; // minimal strength to calculate rmse (ignore weaker) + int indx_used = 2; + int indx_last = 0; + int indx_initial = 3; + int indx_strength = 1; + double max_disparity = 200.0; // for max_err1 + + final double[][] sky_map = ref_scene. readDoubleArrayFromModelDirectory( + "-sky_mask", // String suffix, + 0, // int num_slices, // (0 - all) + null); // int [] wh); + + double [][] ref_dsn = ref_scene.readDoubleArrayFromModelDirectory( + "-results-nonoise", // String suffix, + 0, // int num_slices, // (0 - all) + null); // int [] wh); + int indx_last_diff = ref_dsn.length - 1; + boolean [] good_tiles = new boolean [ref_dsn[0].length]; + int num_good_init = 0; + for (int i = 0; i < good_tiles.length; i++) { + good_tiles[i] = (ref_dsn[indx_used][i] > 0.999) && (Math.abs(ref_dsn[indx_last_diff][i]) < max_diff); + if (good_tiles[i] && (sky_map != null) && (sky_map[0][i] > 0.0)) { + good_tiles[i] = false; + } + if (good_tiles[i]){ + num_good_init++; + } + } +// double [] noise_level = new double [noise_files.length]; +// boolean [] intra = new boolean [noise_files.length]; +// boolean [] inter = new boolean [noise_files.length]; + class DisparityResults{ + double [][] results; + } + HashMap results_map = new HashMap (); + for (int nf = 0; nf < noise_files.length; nf++) { + String fn = noise_files[nf]; + String [] tokens = fn.replace("E-","E_minus").split("-"); + double noise_level = Double.parseDouble(tokens[2].replace("E_minus","E-").substring("lev_".length())); + boolean inter = !fn.contains("nointer"); + boolean intra = !fn.contains("mask1"); +// System.out.println("level="+noise_level+", inter="+inter+", intra="+intra); + + double [][] noise_dsn = ref_scene.readDoubleArrayFromModelDirectory( + fn, // noise_files[nf], // String suffix, + 0, // int num_slices, // (0 - all) + null); // int [] wh); + boolean [] converged_tiles_this = good_tiles.clone(); + boolean [] good_tiles_this = good_tiles.clone(); + boolean [] good_tiles_this1 = good_tiles.clone(); + int num_converged = 0; + int num_good = 0; + int num_good1 = 0; + int num_near = 0; + int num_converged_near = 0; + + + double s0 = 0.0; + double s2 = 0.0; + for (int i = 0; i < good_tiles.length; i++) if (good_tiles[i]) { + converged_tiles_this[i] = (Math.abs(noise_dsn[indx_last_diff][i]) < max_diff); + if (converged_tiles_this[i]){ + num_converged++; + good_tiles_this[i] = (Math.abs(noise_dsn[indx_last][i] - noise_dsn[indx_initial][i]) < max_err); + if (good_tiles_this[i]) { + num_good++; + double w = (noise_dsn[indx_strength][i] < min_strength)? 0.0 : 1.0; + s0 += w; + double d = noise_dsn[indx_last][i] - noise_dsn[indx_initial][i]; + s2 += w * d * d; + } + /* + good_tiles_this1[i] = (Math.abs(noise_dsn[indx_last][i] - noise_dsn[indx_initial][i]) < max_err1); + if (good_tiles_this1[i]) { + num_good1++; + } + */ + } + if (noise_dsn[indx_initial][i] <= max_disparity) { // only for near tiles + num_near++; + if (converged_tiles_this[i]){ + num_converged_near++; + good_tiles_this1[i] = (Math.abs(noise_dsn[indx_last][i] - noise_dsn[indx_initial][i]) < max_err1); + if (good_tiles_this1[i]) { + num_good1++; + } + } + } + } + double rmse = Math.sqrt(s2/s0); + /* + * int indx_strength = 1; + + double perc_good = 100.0* num_good/num_good_init; + double perc_good1 = 100.0* num_good1/num_good_init; + double perc_good_conf = 100.0* num_good/num_converged; + */ + double [] results = { + 1.0* num_good/num_good_init, +// 1.0* num_good1/num_good_init, + 1.0* num_good1/num_near, + 1.0* num_good/num_converged, + 1.0* num_good1/num_converged_near, + rmse + }; + if (!results_map.containsKey(noise_level)) { + DisparityResults dr = new DisparityResults(); + dr.results = new double [3][]; + results_map.put(noise_level, dr); + } + int results_index = inter ? 0 : (intra? 1 : 2); + DisparityResults dr = results_map.get(noise_level); + dr.results[results_index] = results; + System.out.println("getNoiseStats(): "+noise_files[nf]+": good_ref= " + num_good_init+ +// ", converged= "+num_converged+", good= "+num_good+", good(0.5)= "+perc_good+"%, good(0.1)= "+perc_good1+"%, perc_good_conf= "+perc_good_conf+"%"); + ", converged= "+num_converged+", good= "+num_good+" good(0.1)= "+num_good1+", num_near="+num_near); + } + List noise_levels_list = new ArrayList(results_map.keySet()); + Collections.sort(noise_levels_list); + System.out.println("\n"); + System.out.print("noise_level, "); + System.out.print("inter("+max_err+"), inter("+max_err1+"), inter_conf("+max_err+"), inter_conf("+max_err1+"), inter_rmse("+max_err+"),"); + System.out.print("intra("+max_err+"), intra("+max_err1+"), intra_conf("+max_err+"), intra_conf("+max_err1+"), intra_rmse("+max_err+"),"); + System.out.print("binocular("+max_err+"), binocular("+max_err1+"), binocular_conf("+max_err+"), binocular_conf("+max_err1+"), binocular_rmse("+max_err+")"); + System.out.println(); + for (Double nl:noise_levels_list) { + System.out.print(nl+", "); + double [][] results = results_map.get(nl).results; + for (int n = 0; n < results.length; n++) { + for (int i = 0; i < results[n].length; i++) { + System.out.print(results[n][i]); + if ((n < (results.length -1)) ||(i< (results[n].length - 1))) { + System.out.print(", "); + } + } + } + System.out.println(); + } + System.out.println(); + } + + public void batchLwirRig( QuadCLT quadCLT_main, // tiles should be set @@ -8791,9 +9226,10 @@ if (debugLevel > -100) return true; // temporarily ! channelGainParameters, rgbParameters, // EyesisCorrectionParameters.RGBParameters rgbParameters, scaleExposures_main, // double [] scaleExposures, // probably not needed here - restores brightness of the final image + false, // boolean only4slice, threadsMax, // final int threadsMax, // maximal number of threads to launch updateStatus, // final boolean updateStatus, - debugLevel); // final int debugLevel); + debugLevel); // final int debugLevel); } else { if (updateStatus) IJ.showStatus("CPU: Rendering 4 image set (disparity = 0) for "+quadCLT_main.image_name+ "and a thumb nail"); -- 2.18.1