Working on FOPEN (8e12622f) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/orthomosaic/VegetationModel.java

0 → 100644

+1197 −0

File added.

Preview size limit exceeded, changes collapsed.

src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java

+18 −0

Original line number	Diff line number	Diff line
		@@ -60,6 +60,7 @@ import com.elphel.imagej.ims.Did_ins_2;
		import com.elphel.imagej.ims.Did_pimu;
		import com.elphel.imagej.ims.Imx5;
		import com.elphel.imagej.jp4.JP46_Reader_camera;
		import com.elphel.imagej.orthomosaic.VegetationModel;
		import com.elphel.imagej.tileprocessor.sfm.StructureFromMotion;

		import Jama.Matrix;
		@@ -5685,6 +5686,23 @@ public class OpticalFlow {

		// generates 3-d modes, colors, stereos, tiffs/videos

		// Testing vegetation, for debugging supposing that terrain layer is already set in *-INTER-INTRA-LMA.tiff - normally it is only set during 3d model generation
		boolean test_vegetation = true;
		if (test_vegetation) { // limit start of the quadCLTs by reading start/end from the reference scene
		VegetationModel.test_vegetation(
		clt_parameters, // CLTParameters clt_parameters,
		quadCLTs, // QuadCLT [] quadCLTs,
		ref_index, // int ref_index,
		debugLevel); // int debugLevel)
		if (videos != null) {
		videos[0] = new String[0];
		}
		// temporarily - exiting now
		return quadCLTs[ref_index].getX3dTopDirectory();
		}



		if (generate_mapped \|\| reuse_video) { // modifies combo_dsn_final ?
		int tilesX = quadCLTs[ref_index].getTileProcessor().getTilesX();
		int tilesY = quadCLTs[ref_index].getTileProcessor().getTilesY();

src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java

+218 −0

Original line number	Diff line number	Diff line
		@@ -1471,6 +1471,172 @@ public class QuadCLT extends QuadCLTCPU {
		return imp_render;
		}


		public static double [][] getScenePxPyD(
		final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
		double [] disparity_ref,
		double [] scene_xyz, // camera center in world coordinates. If null - no shift, no ers
		double [] scene_atr, // camera orientation relative to world frame
		final QuadCLT scene,
		final QuadCLT ref_scene){ // now - may be null - for testing if scene is rotated ref

		double [][] pXpYD;
		if ((scene_xyz == null) \|\| (scene_atr == null)) {
		scene_xyz = new double[3];
		scene_atr = new double[3];
		pXpYD=OpticalFlow.transformToScenePxPyD( // now should work with offset ref_scene
		full_woi_in, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
		disparity_ref, // final double [] disparity_ref, // invalid tiles - NaN in disparity
		scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
		scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
		ref_scene, // final QuadCLT scene_QuadClt,
		ref_scene, // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
		THREADS_MAX); // int threadsMax)
		} else {
		pXpYD=OpticalFlow.transformToScenePxPyD( // now should work with offset ref_scene
		full_woi_in, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
		disparity_ref, // final double [] disparity_ref, // invalid tiles - NaN in disparity
		scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
		scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
		scene, // final QuadCLT scene_QuadClt,
		ref_scene, // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
		THREADS_MAX); // int threadsMax)
		}
		return pXpYD;

		}

		public static double [][] renderDoubleGPUFromDSI( // {scene}{color}{pixel}
		final int sensor_mask,
		final Rectangle full_woi_in, // show larger than sensor WOI in tiles (or null)
		CLTParameters clt_parameters,
		double [] disparity_ref,
		// motion blur compensation
		double mb_tau, // 0.008; // time constant, sec
		double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
		double [][] mb_vectors, // now [2][ntiles];
		final QuadCLT scene,
		final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
		final boolean show_nan,
		double [][] pXpYD,
		final int debugLevel){
		int rendered_width = scene.getErsCorrection().getSensorWH()[0];
		if (full_woi_in != null) {
		rendered_width = full_woi_in.width * GPUTileProcessor.DTT_SIZE;
		}
		boolean showPxPyD = false;
		if (showPxPyD) {
		int dbg_width = rendered_width/GPUTileProcessor.DTT_SIZE;
		int dbg_height = pXpYD.length/dbg_width;
		double [][] dbg_img = new double [3 + ((mb_vectors!=null)? 2:0)][pXpYD.length];
		String [] dbg_titles = (mb_vectors!=null)?
		(new String[] {"pX","pY","Disparity","mb_X","mb_Y"}):
		(new String[] {"pX","pY","Disparity"});
		for (int i = 0; i < dbg_img.length; i++) {
		Arrays.fill(dbg_img[i], Double.NaN);
		}
		for (int nTile = 0; nTile < pXpYD.length; nTile++){
		if (pXpYD[nTile] != null) {
		for (int i = 0; i < pXpYD[nTile].length; i++) {
		dbg_img[i][nTile] = pXpYD[nTile][i];
		}
		}
		if (mb_vectors!=null) {
		for (int i = 0; i <2; i++) {
		dbg_img[3 + i][nTile] = mb_tau * mb_vectors[i][nTile];
		}
		}
		}
		ShowDoubleFloatArrays.showArrays( // out of boundary 15
		dbg_img,
		dbg_width,
		dbg_height,
		true,
		scene.getImageName()+"-pXpYD",
		dbg_titles);
		}
		TpTask[][] tp_tasks;
		if (mb_vectors!=null) {
		tp_tasks = GpuQuad.setInterTasksMotionBlur( // "true" reference, with stereo actual reference will be offset
		scene.getNumSensors(),
		rendered_width, // should match output size, pXpYD.length
		!scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
		pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
		null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
		// motion blur compensation
		mb_tau, // final double mb_tau, // 0.008; // time constant, sec
		mb_max_gain, // final double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
		mb_vectors, //final double [][] mb_vectors, //
		scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
		clt_parameters.imp.disparity_corr, // 04/07/2023 //0.0, // final double disparity_corr,
		-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
		null, // final boolean [] valid_tiles,
		THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
		} else {
		tp_tasks = new TpTask[1][];
		tp_tasks[0] = GpuQuad.setInterTasks( // "true" reference, with stereo actual reference will be offset
		scene.getNumSensors(),
		rendered_width, // should match output size, pXpYD.length
		!scene.hasGPU(), // final boolean calcPortsCoordinatesAndDerivatives, // GPU can calculate them centreXY
		pXpYD, // final double [][] pXpYD, // per-tile array of pX,pY,disparity triplets (or nulls)
		null, // final boolean [] selection, // may be null, if not null do not process unselected tiles
		scene.getErsCorrection(), // final GeometryCorrection geometryCorrection,
		clt_parameters.imp.disparity_corr, // 04/07/2023 // 0.0, // final double disparity_corr,
		-1, // 0, // margin, // final int margin, // do not use tiles if their centers are closer to the edges
		null, // final boolean [] valid_tiles,
		THREADS_MAX); // final int threadsMax) // maximal number of threads to launch
		}
		scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
		ImageDtt image_dtt = new ImageDtt(
		scene.getNumSensors(),
		clt_parameters.transform_size,
		clt_parameters.img_dtt,
		scene.isAux(),
		scene.isMonochrome(),
		scene.isLwir(),
		clt_parameters.getScaleStrength(scene.isAux()),
		scene.getGPU());
		boolean use_reference = false;
		int [] wh = (full_woi_in == null)? null: new int[]{
		full_woi_in.width * GPUTileProcessor.DTT_SIZE,
		full_woi_in.height * GPUTileProcessor.DTT_SIZE};
		int erase_clt = show_nan ? 1:0;
		// boolean test1 = true;
		if (mb_vectors!=null) {// && test1) {
		image_dtt.setReferenceTDMotionBlur( // change to main?
		erase_clt, //final int erase_clt,
		wh, // null, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
		clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
		use_reference, // true, // final boolean use_reference_buffer,
		tp_tasks, // final TpTask[] tp_tasks,
		clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
		clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
		clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
		clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
		THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
		debugLevel); // final int globalDebugLevel);
		} else {
		image_dtt.setReferenceTD( // change to main?
		erase_clt, //final int erase_clt,
		wh, // null, // final int [] wh, // null (use sensor dimensions) or pair {width, height} in pixels
		clt_parameters.img_dtt, // final ImageDttParameters imgdtt_params, // Now just extra correlation parameters, later will include, most others
		use_reference, // true, // final boolean use_reference_buffer,
		tp_tasks[0], // final TpTask[] tp_tasks,
		clt_parameters.gpu_sigma_r, // final double gpu_sigma_r, // 0.9, 1.1
		clt_parameters.gpu_sigma_b, // final double gpu_sigma_b, // 0.9, 1.1
		clt_parameters.gpu_sigma_g, // final double gpu_sigma_g, // 0.6, 0.7
		clt_parameters.gpu_sigma_m, // final double gpu_sigma_m, // = 0.4; // 0.7;
		THREADS_MAX, // final int threadsMax, // maximal number of threads to launch
		debugLevel); // final int globalDebugLevel);
		}
		double [][] double_render = scene.renderDoubleFromTD (
		sensor_mask, // final int sensor_mask,
		wh, // null, // int [] wh,
		use_reference); // boolean use_reference
		return double_render;
		}


		/**
		* Prepare 16x16 texture tiles using GPU from disparity reference. Includes motion blur correction
		* Does not use scene.saveQuadClt() to re-load new set of Bayer images to the GPU -- should be done by caller
		@@ -2264,6 +2430,58 @@ public class QuadCLT extends QuadCLTCPU {
		return imp_stack;
		}

		/**
		* Render single scene, combining all sensors as a double array [colorchn][pixel]
		* @param sensor_mask sensors to use (normally -1 - all)
		* @param clt_parameters
		* @param wh null or {width, height}
		* @param use_reference
		* @return
		*/
		public double [][] renderDoubleFromTD ( // [scene][color][pixel]
		int sensor_mask,
		// CLTParameters clt_parameters,
		int [] wh, // may be null, or {width, height}
		boolean use_reference
		) {
		gpuQuad.execImcltRbgAll(
		isMonochrome(),
		use_reference,
		wh); //int [] wh
		// get data back from GPU
		final float [][][] iclt_fimg = new float [getNumSensors()][][];
		int nchn = 0;
		int ncol = 0;
		int nTiles = 0;
		for (int ncam = 0; ncam < iclt_fimg.length; ncam++) if (((1 << ncam) & sensor_mask) != 0){
		iclt_fimg[ncam] = gpuQuad.getRBG(ncam); // updated window
		ncol = iclt_fimg[ncam].length;
		nTiles = iclt_fimg[ncam][0].length;
		nchn++;
		}
		final double scale = 1.0 / nchn;
		final double [][] iclt_fimg_combo = new double [ncol][nTiles];
		final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
		final AtomicInteger ai = new AtomicInteger(0);
		for (int ithread = 0; ithread < threads.length; ithread++) {
		threads[ithread] = new Thread() {
		public void run() {
		for (int nTile = ai.getAndIncrement(); nTile < iclt_fimg_combo[0].length; nTile = ai.getAndIncrement()) {
		for (int ncol = 0; ncol < iclt_fimg_combo.length; ncol++) {
		double d = 0;
		for (int i = 0; i < iclt_fimg.length; i++) if (iclt_fimg[i] != null) {
		d+=iclt_fimg[i][ncol][nTile];
		}
		iclt_fimg_combo[ncol][nTile] = d * scale; // (float) (d * scale);
		}
		}
		}
		};
		}
		ImageDtt.startAndJoin(threads);
		return iclt_fimg_combo;
		}

src/main/java/com/elphel/imagej/tileprocessor/TexturedModel.java

+1 −101

Original line number	Diff line number	Diff line
		@@ -3044,96 +3044,6 @@ public class TexturedModel {

		}

		/*

		if ((smooth_ground != null) && !ct_mode) { // no sense to use with CT? or compare with the common only?



		if (has_sfm && gsmth_sfm_deviate) {
		mixed_flat = true;
		// Change!
		// Use FG for FG, and ground plane as BG?
		TileNeibs tn = new TileNeibs(tilesX, smooth_ground.length/tilesX);
		for (int ntile = 0; ntile < smooth_ground.length; ntile++) {
		double disp = (ss_fg_bg[1][ntile] > ss_fg_bg[0][ntile]) ? ds_fg_bg[1][ntile] : ds_fg_bg[1][ntile]; // strongest
		if (disp > lim_disparity) {
		disp = lim_disparity;
		}

		double str = Math.max(ss_fg_bg[1][ntile], ss_fg_bg[0][ntile]);
		boolean strong = false;
		if (disp > smooth_ground[ntile]) {
		strong = str >= gsmth_str_bump;
		} else {
		strong = str >= gsmth_str_dip;
		}
		if (strong) {
		check_neibs:
		{
		for (int dy = -gsmth_sfm_rad; dy <= gsmth_sfm_rad; dy++) {
		for (int dx = -gsmth_sfm_rad; dx <= gsmth_sfm_rad; dx++) {
		int ntile1 = tn.getNeibIndex(ntile, dx, dy);
		if ((ntile1 < 0) \|\| (sfm_gain[ntile1] < gsmth_sfm_gain)) {
		strong = false;
		break check_neibs;
		}
		}
		}
		if (disp > smooth_ground[ntile]) {
		ds_fg_bg[0][ntile] = disp;
		ss_fg_bg[0][ntile] = str;
		ds_fg_bg[1][ntile] = smooth_ground[ntile];
		ss_fg_bg[1][ntile] = small_strength;
		} else {
		ds_fg_bg[0][ntile] = Double.NaN;
		ss_fg_bg[0][ntile] = 0.0;
		smooth_ground[ntile] = disp;
		ds_fg_bg[1][ntile] = smooth_ground[ntile];
		ss_fg_bg[1][ntile] = str;
		}
		}
		}
		if (!strong) { // either was weak or broke check_neibs
		// if (ct_mode) {
		// ds_fg_bg[0][ntile] = lim_disparity;
		// ss_fg_bg[0][ntile] = str;
		// ds_fg_bg[1][ntile] = smooth_ground[ntile];
		// ss_fg_bg[1][ntile] = small_strength;
		// } else {
		ds_fg_bg[0][ntile] = Double.NaN;
		ds_fg_bg[1][ntile] = smooth_ground[ntile];
		ss_fg_bg[0][ntile] = 0.0;
		ss_fg_bg[1][ntile] = str;
		// }
		}
		}
		} else { // smooth surface only: nothing in FG, surface - in BG
		if (!ct_mode) { // no sense to use smooth_ground in CT scan mode
		for (int i = 0; i < smooth_ground.length; i++) {
		ds_fg_bg[1][i] = smooth_ground[i];
		ss_fg_bg[1][i] = Math.max(ss_fg_bg[1][i],ss_fg_bg[0][i]);
		ss_fg_bg[1][i] = Math.max(ss_fg_bg[1][i],small_strength);
		ds_fg_bg[0][i] = Double.NaN;
		ss_fg_bg[0][i] = 0.0;
		}
		}

		}

		if (show_bs_debug && (debugLevel>1)) {
		String [] dbg_titles = {"FGC","BGC","FG", "BG", "GND", "FS", "BS"};
		double [][] dbg_img = {ds_fg_bg_common[0], ds_fg_bg_common[1], ds_fg_bg[0], ds_fg_bg[1], smooth_ground, ss_fg_bg_common[0], ss_fg_bg_common[1]}; // here BG==FG
		ShowDoubleFloatArrays.showArrays(
		dbg_img,
		tilesX,
		dbg_img[0].length/tilesX,
		true,
		ref_scene.getImageName()+"-fgc-bgc-fg-bg-disparity_strength-LMA-GND",
		dbg_titles);
		}
		}
		*/
		if (show_bs_debug) {
		String [] dbg_titles = {"FGC","BGC","FG", "BG", "GND", "FS", "BS"};
		double [][] dbg_img = {ds_fg_bg_common[0], ds_fg_bg_common[1], ds_fg_bg[0], ds_fg_bg[1], smooth_ground, ss_fg_bg_common[0], ss_fg_bg_common[1]}; // here BG==FG
		@@ -3248,10 +3158,9 @@ public class TexturedModel {
		min_trim_disparity, // final double min_trim_disparity, // 2.0; // do not try to trim texture outlines with lower disparities
		lin_text0, // final double [][][][] lin_textures, // null or [1][][][] to return non-normalized textures
		(ct_scans != null), // ct_mode, // final boolean ct_mode,
		terrain_mode, // final boolean terrain_mode, Just for file name
		terrain_mode, // final boolean terrain_mode, just for file name
		debugLevel); // final int debug_level)
		if (ct_scans != null) {
		// ct_scans[nscan] = new double [][] {faded_textures[0][0].clone(),faded_textures[1][0].clone()}; // both slices, channel 0
		ct_scans[nscan] = new double [faded_textures.length][];
		for (int i = 0; i < ct_scans[nscan].length; i++) {
		ct_scans[nscan][i] = faded_textures[i][0].clone(); // channel 0 from each slice
		@@ -3266,15 +3175,6 @@ public class TexturedModel {
		titles[nscan] = String.format("D=%.3f", ct_min+ct_step*nscan);
		}
		titles[titles.length-1]="full";
		/*
		ShowDoubleFloatArrays.showArrays(
		ct_disparity,
		tilesX,
		ct_disparity[0].length/tilesX,
		true,
		ref_scene.getImageName()+"-CT_scan_disparities",
		titles);
		*/
		double ct_max = ct_min+ct_step* (ct_scans.length-1);
		String suffix = String.format("-CT_SCAN_%.3f_%.3f_%.3f",ct_min,ct_max,ct_step);
		scenes[ref_index].saveDoubleArrayInModelDirectory(

src/main/java/com/elphel/imagej/tileprocessor/TwoQuadCLT.java

+3 −0

Original line number	Diff line number	Diff line
		@@ -8660,6 +8660,9 @@ if (debugLevel > -100) return true; // temporarily !
		}
		first_in_series = false; // at least once success in this series
		System.out.println("adding to video_sets_list start="+start_ref_pointers[0]+", end="+ref_index); // start_ref_pointers[1]);
		// if ((video_list != null) && (video_list.length>0) && (video_list[0] == null)) {
		// video_list[0]= new String[0];
		// }
		for (int i= 0; i < video_list[0].length; i++) {
		System.out.println(i+": "+video_list[0][i]+", "+widths_list[0][i]);
		}

src/main/java/com/elphel/imagej/orthomosaic/OrangeTest.java

+1 −1

File changed.

Contains only whitespace changes.

Admin message