Implemented CLT accumulation for virtual orientation (c216c7e4) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/gpu/GpuQuad.java

+47 −15

Original line number	Original line	Diff line number	Diff line
	@@ -1114,6 +1114,17 @@ public class GpuQuad{ // quad camera description
	copyD2H.Height = img_height; // /4;		copyD2H.Height = img_height; // /4;
	cuMemcpy2D(copyD2H); // run copy		cuMemcpy2D(copyD2H); // run copy
	}		}
			public int [] getWH(boolean use_ref) {
			return use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
			}

			public int getNumColors() {
			return num_colors;
			}

			public int getNumSensors() {
			return num_cams;
			}

	public int getCltSize(boolean use_ref) { // per camera, in floats		public int getCltSize(boolean use_ref) { // per camera, in floats
	int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;		int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
	@@ -1121,7 +1132,16 @@ public class GpuQuad{ // quad camera description
	int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;		int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;
	return tilesYtilesXnum_colors* 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;		return tilesYtilesXnum_colors* 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
	}		}
			/*
			public int getCltLength(boolean use_ref) {
			int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
			int tilesX = wh[0] / GPUTileProcessor.DTT_SIZE;
			int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;
			int tile_size_td = getCltSize(use_ref); // 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
			int num_tiles = tilesYtilesXnum_colors;
			return num_tiles* tile_size_td;
			}
			*/
	public float [][] getCltData( // only for color=0		public float [][] getCltData( // only for color=0
	boolean use_ref){		boolean use_ref){
	CUdeviceptr [] gpu_sel_clt_h = use_ref ? gpu_clt_ref_h : gpu_clt_h;		CUdeviceptr [] gpu_sel_clt_h = use_ref ? gpu_clt_ref_h : gpu_clt_h;
	@@ -1209,24 +1229,36 @@ public class GpuQuad{ // quad camera description
	return;		return;
	}		}

	/*		public void setCltData( // for testing only
	public void setBayerImage(		int ncam,
	float [] bayer_image,		float [] fclt, //
	int ncam) {		boolean use_ref){
			int clt_size = getCltSize(use_ref);
			if (fclt.length != clt_size) {
			System.out.println("getCltData(): wrong array size: got ["+fclt.length+"], "+
			"should be ["+clt_size+"]");
			return;
			}

			CUdeviceptr [] gpu_sel_clt_h = use_ref ? gpu_clt_ref_h : gpu_clt_h;
			int [] wh = use_ref ? gpu_clt_ref_wh : gpu_clt_wh;
			int tilesX = wh[0] / GPUTileProcessor.DTT_SIZE;
			int tilesY = wh[1] / GPUTileProcessor.DTT_SIZE;
			int tile_size_td = 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
			int num_tiles = tilesYtilesXnum_colors;
	CUDA_MEMCPY2D copyH2D = new CUDA_MEMCPY2D();		CUDA_MEMCPY2D copyH2D = new CUDA_MEMCPY2D();
	copyH2D.srcMemoryType = CUmemorytype.CU_MEMORYTYPE_HOST;		copyH2D.srcMemoryType = CUmemorytype.CU_MEMORYTYPE_HOST;
	copyH2D.srcHost = Pointer.to(bayer_image);		copyH2D.srcHost = Pointer.to(fclt);
	copyH2D.srcPitch = img_width*Sizeof.FLOAT; // width_in_bytes;		copyH2D.srcPitch = tile_size_td * Sizeof.FLOAT;
	copyH2D.dstMemoryType = CUmemorytype.CU_MEMORYTYPE_DEVICE;		copyH2D.dstMemoryType = CUmemorytype.CU_MEMORYTYPE_DEVICE;
	copyH2D.dstDevice = gpu_bayer_h[ncam]; // src_dpointer;		copyH2D.dstDevice = gpu_sel_clt_h[ncam];
	copyH2D.dstPitch = mclt_stride *Sizeof.FLOAT; // device_stride[0];		copyH2D.dstPitch = tile_size_td * Sizeof.FLOAT;
	copyH2D.WidthInBytes = img_width*Sizeof.FLOAT; // width_in_bytes;		copyH2D.WidthInBytes = tile_size_td * Sizeof.FLOAT;
	copyH2D.Height = img_height; // /4;		copyH2D.Height = num_tiles;
	cuMemcpy2D(copyH2D);		cuMemcpy2D(copyH2D); // run copy
			return;
	}		}

	*/


	/**		/**
	* Copy a set of images to the GPU (if they are new)		* Copy a set of images to the GPU (if they are new)

src/main/java/com/elphel/imagej/tileprocessor/OpticalFlow.java

+266 −10

Original line number	Original line	Diff line number	Diff line
	@@ -55,6 +55,7 @@ import com.elphel.imagej.common.PolynomialApproximation;
	import com.elphel.imagej.common.ShowDoubleFloatArrays;		import com.elphel.imagej.common.ShowDoubleFloatArrays;
	import com.elphel.imagej.correction.CorrectionColorProc;		import com.elphel.imagej.correction.CorrectionColorProc;
	import com.elphel.imagej.cuas.CuasCenterLma;		import com.elphel.imagej.cuas.CuasCenterLma;
			import com.elphel.imagej.gpu.GPUTileProcessor;
	import com.elphel.imagej.gpu.GpuQuad;		import com.elphel.imagej.gpu.GpuQuad;
	import com.elphel.imagej.gpu.TpTask;		import com.elphel.imagej.gpu.TpTask;
	import com.elphel.imagej.ims.Did_ins_1;		import com.elphel.imagej.ims.Did_ins_1;
	@@ -5914,6 +5915,67 @@ public class OpticalFlow {
	cuas_atr = new double [] { center_ATR[0][0], center_ATR[0][1], center_ATR[0][2]};		cuas_atr = new double [] { center_ATR[0][0], center_ATR[0][1], center_ATR[0][2]};
	}		}

			boolean combine_clt = true;
			if (combine_clt) {
			boolean apply_clt = true; // set GPU with data
			boolean show_clt = true;
			boolean merge_clt = true;
			QuadCLT ref_clt = quadCLTs[ref_index];
			int sensor_mask_clt = -1; // all

			if (combo_dsn_final == null) {
			combo_dsn_final =quadCLTs[ref_index].restoreComboDSI(true); // also sets quadCLTs[ref_index].dsi and blue sky
			}
			double [][] dls = {
			combo_dsn_final[COMBO_DSN_INDX_DISP],
			combo_dsn_final[COMBO_DSN_INDX_LMA],
			combo_dsn_final[COMBO_DSN_INDX_STRENGTH]
			};
			double [][] ds_clt = conditionInitialDS(
			true, // boolean use_conf, // use configuration parameters, false - use following
			clt_parameters, // CLTParameters clt_parameters,
			dls, // double [][] dls
			quadCLTs[ref_index], // QuadCLT scene,
			debugLevel);

			float [][] combo_seq_clt = getTDComboSceneSequence(
			clt_parameters, // CLTParameters clt_parameters,
			merge_clt, // boolean merge_all,
			sensor_mask_clt, // int sensor_mask,
			null, // Rectangle fov_tiles,
			ZERO3, // double [] stereo_xyz, // offset reference camera {x,y,z}
			cuas_atr, // double [] stereo_atr_in, // offset reference orientation (cuas)
			ds_clt[0], // double [] ref_disparity,
			quadCLTs, // QuadCLT [] quadCLTs,
			ref_clt, // QuadCLT refCLT, // should be the same instance if one of quadCLTs
			debugLevel); // int debugLevel)
			int [] whc = new int[3];
			if (apply_clt) { // set GPU with data
			quadCLTs[ref_index].setComboToTD(
			combo_seq_clt, // final float [][] fclt,
			merge_clt, // final boolean merge_channels, // duplicate same data to all selected channels
			sensor_mask_clt, // final int sensor_mask, // only if merge_channels
			whc, // final int [] whc, // if int[2], will return width, height
			false); // final boolean use_reference);
			if (show_clt) {
			String suffix="-virtual";
			ImagePlus imp_virtual = ref_clt.renderFromTD ( // do we need to update gpuQuad ?
			sensor_mask_clt, // final int sensor_mask,
			merge_clt, // boolean merge_channels,
			clt_parameters, // CLTParameters clt_parameters,
			clt_parameters.getColorProcParameters(ref_clt.isAux()), //ColorProcParameters colorProcParameters,
			clt_parameters.getRGBParameters(), //EyesisCorrectionParameters.RGBParameters rgbParameters,\
			whc, // null, // int [] wh,
			false, // toRGB, // boolean toRGB,
			false, // use_reference, // boolean use_reference
			suffix); // String suffix)
			imp_virtual.show();
			}
			}
			}



	if (generate_egomotion) {		if (generate_egomotion) {
	boolean ego_show = !clt_parameters.batch_run; //true;		boolean ego_show = !clt_parameters.batch_run; //true;
	String ego_path = quadCLTs[ref_index].getX3dDirectory()+Prefs.getFileSeparator()+		String ego_path = quadCLTs[ref_index].getX3dDirectory()+Prefs.getFileSeparator()+
	@@ -7859,16 +7921,6 @@ public class OpticalFlow {
	double [][] ref_pXpYD;		double [][] ref_pXpYD;
	double [][] ref_pXpYD_or_null = null; // debugging cuas mode keeping old		double [][] ref_pXpYD_or_null = null; // debugging cuas mode keeping old
	if (mode_cuas) { // && (dbg_scene > 0)) {		if (mode_cuas) { // && (dbg_scene > 0)) {
	/*
	double [][] ref_pXpYD_0 = transformToScenePxPyD( // now should work with offset ref_scene
	fov_tiles, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
	ref_disparity, // final double [] disparity_ref, // invalid tiles - NaN in disparity
	ZERO3, // stereo_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
	ZERO3, // stereo_atr, // ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
	quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
	quadCLTs[ref_index], // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
	threadsMax); // int threadsMax)
	*/
	int around = 2;		int around = 2;
	double around_sigma = 4.0;		double around_sigma = 4.0;
	int num_virtual_refines = 2;		int num_virtual_refines = 2;
	@@ -8115,6 +8167,210 @@ public class OpticalFlow {
	return imp_scenes;		return imp_scenes;
	}		}

			public static float [][] getTDComboSceneSequence(
			CLTParameters clt_parameters,
			boolean merge_all,
			int sensor_mask,
			Rectangle fov_tiles,
			double [] stereo_xyz, // offset reference camera {x,y,z}
			double [] stereo_atr_in, // offset reference orientation (cuas)
			double [] ref_disparity,
			QuadCLT [] quadCLTs,
			QuadCLT refCLT, // should be the same instance if one of quadCLTs
			// int ref_index,
			int debugLevel) {
			double [] stereo_atr = (stereo_atr_in != null)? stereo_atr_in: ZERO3; // maybe later play with rotated camera
			boolean mode_cuas = (stereo_atr[0] != 0) \|\| (stereo_atr[1] != 0) \|\| (stereo_atr[2] != 0);
			boolean mb_en = clt_parameters.imp.mb_en && (fov_tiles==null);
			double mb_tau = clt_parameters.imp.mb_tau; // 0.008; // time constant, sec
			double mb_max_gain = clt_parameters.imp.mb_max_gain; // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
			ErsCorrection ers_reference = refCLT.getErsCorrection();
			int dbg_scene = -95;
			double [][] ref_pXpYD;
			double [][] ref_pXpYD_or_null = null; // debugging cuas mode keeping old
			if (mode_cuas) { // && (dbg_scene > 0)) {
			int around = 2;
			double around_sigma = 4.0;
			int num_virtual_refines = 2;
			String debugSuffix= null; // "virtual";
			ref_pXpYD= transformFromVirtual(
			ref_disparity, // double [] disparity_ref,
			stereo_xyz, // final double [] scene_xyz, // camera center in world (reference) coordinates
			stereo_atr, // final double [] scene_atr, // camera orientation relative to world (reference) frame
			refCLT, // quadCLTs[ref_index], // final QuadCLT reference_QuadClt,
			around, // final int around, // 2 search around for interpolation
			around_sigma, // final double sigma,
			num_virtual_refines, // final int num_refines,
			debugSuffix); // final String debugSuffix)
			ref_pXpYD_or_null = ref_pXpYD;
			// quadCLTs[ref_index].getErsCorrection().setupERS();
			refCLT.getErsCorrection().setupERS();
			System.out.println("Calculated virtual_PxPyD");
			} else {
			ref_pXpYD = transformToScenePxPyD( // now should work with offset ref_scene
			fov_tiles, // final Rectangle [] extra_woi, // show larger than sensor WOI (or null)
			ref_disparity, // final double [] disparity_ref, // invalid tiles - NaN in disparity
			ZERO3, // stereo_xyz, // ZERO3, // final double [] scene_xyz, // camera center in world coordinates
			ZERO3, // stereo_atr, // ZERO3, // final double [] scene_atr, // camera orientation relative to world frame
			refCLT, // quadCLTs[ref_index], // final QuadCLT scene_QuadClt,
			refCLT, // quadCLTs[ref_index], // final QuadCLT reference_QuadClt, // now - may be null - for testing if scene is rotated ref
			QuadCLT.THREADS_MAX); // int threadsMax)
			}
			// quadCLTs[0].getCltLength(boolean use_ref)
			int sc0 = -1;
			for (int nscene = 0; nscene < quadCLTs.length ; nscene++) if (quadCLTs[nscene] != null){
			sc0 = nscene;
			break;
			}
			final float [][] sumFclt = new float [merge_all? 1 : quadCLTs[sc0].getNumSensors()][quadCLTs[sc0].getCltSize(false)];
			final int [][] numAcc = new int [sumFclt.length][sumFclt[0].length];
			// next two to improve multithreading performance
			final int tile_size_td = 4 * GPUTileProcessor.DTT_SIZE * GPUTileProcessor.DTT_SIZE;
			final int tiles_td = sumFclt[0].length/tile_size_td;
			final int tiles_td_all = tiles_td * sumFclt.length; // usually sumFclt.length==1

			final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
			final AtomicInteger ai = new AtomicInteger(0);
			for (int nscene = 0; nscene < quadCLTs.length ; nscene++) if (quadCLTs[nscene] != null){
			if (nscene== dbg_scene) {
			System.out.println("renderSceneSequence(): nscene = "+nscene);
			}
			String ts = quadCLTs[nscene].getImageName();
			double [] scene_xyz = ZERO3;
			double [] scene_atr = ZERO3;
			// if (nscene != ref_index) { // Check even for raw, so video frames will match in all modes
			if (quadCLTs[nscene] != refCLT) { // Check even for raw, so video frames will match in all modes
			scene_xyz = ers_reference.getSceneXYZ(ts);
			scene_atr = ers_reference.getSceneATR(ts);
			if ((scene_atr==null) \|\| (scene_xyz == null)) {
			continue;
			}
			double [] scene_ers_xyz_dt = ers_reference.getSceneErsXYZ_dt(ts);
			double [] scene_ers_atr_dt = ers_reference.getSceneErsATR_dt(ts);
			quadCLTs[nscene].getErsCorrection().setErsDt(
			scene_ers_xyz_dt, // double [] ers_xyz_dt,
			scene_ers_atr_dt); // double [] ers_atr_dt)(ers_scene_original_xyz_dt);
			}
			if (!mode_cuas && (stereo_xyz != null)) { // offset all, including reference scene - now always, it is never null
			double [][] combo_xyzatr = ErsCorrection.combineXYZATR(
			stereo_xyz, // double [] reference_xyz,
			stereo_atr, // double [] reference_atr,
			scene_xyz, // double [] scene_xyz,
			scene_atr); // double [] scene_atr)
			scene_xyz = combo_xyzatr[0];
			scene_atr = combo_xyzatr[1];
			}
			int sm = merge_all? -1: sensor_mask;
			float [][] fclt = null;
			double [][] dxyzatr_dt = null;
			// should get velocities from HashMap at reference scene from timestamp , not re-calculate.
			if (mb_en) {
			// dxyzatr_dt = getVelocities(
			// quadCLTs, // QuadCLT [] quadCLTs,
			// nscene); // int nscene)
			dxyzatr_dt = new double[][] { // for all, including ref
			quadCLTs[nscene].getErsCorrection().getErsXYZ_dt(),
			quadCLTs[nscene].getErsCorrection().getErsATR_dt()};
			}


			if (mb_en && (dxyzatr_dt != null)) {
			double [][] motion_blur = getMotionBlur(
			refCLT, // quadCLTs[ref_index], // QuadCLT ref_scene,
			quadCLTs[nscene], // QuadCLT scene, // can be the same as ref_scene
			ref_pXpYD, // double [][] ref_pXpYD, // here it is scene, not reference!
			scene_xyz, // double [] camera_xyz,
			scene_atr, // double [] camera_atr,
			dxyzatr_dt[0], // double [] camera_xyz_dt,
			dxyzatr_dt[1], // double [] camera_atr_dt,
			0, // int shrink_gaps, // will gaps, but not more that grow by this
			debugLevel); // int debug_level)
			fclt = QuadCLT.getTDCombo(
			sm, // final int sensor_mask,
			merge_all, // final boolean merge_channels,
			null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
			clt_parameters, // CLTParameters clt_parameters,
			ref_disparity, // double [] disparity_ref,
			ref_pXpYD_or_null, // double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
			// motion blur compensation
			mb_tau, // double mb_tau, // 0.008; // time constant, sec
			mb_max_gain, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
			motion_blur, // double [][] mb_vectors, //
			scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
			scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
			quadCLTs[nscene], // final QuadCLT scene,
			refCLT, // quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
			true, // final boolean show_nan,
			QuadCLT.THREADS_MAX, // int threadsMax,
			debugLevel); // final int debugLevel)

			} else {
			fclt = QuadCLT.getTDCombo(
			sm, // final int sensor_mask,
			merge_all, // final boolean merge_channels,
			null, // final Rectangle full_woi_in, // show larger than sensor WOI (or null)
			clt_parameters, // CLTParameters clt_parameters,
			ref_disparity, // double [] disparity_ref,
			ref_pXpYD_or_null, // double [][] ref_pXpYD, // alternative to disparity_ref when reference is not uniform
			// motion blur compensation
			0, // double mb_tau, // 0.008; // time constant, sec
			0, // double mb_max_gain, // 5.0; // motion blur maximal gain (if more - move second point more than a pixel
			null, // double [][] mb_vectors, //
			scene_xyz, // final double [] scene_xyz, // camera center in world coordinates
			scene_atr, // final double [] scene_atr, // camera orientation relative to world frame
			quadCLTs[nscene], // final QuadCLT scene,
			refCLT, // quadCLTs[ref_index], // final QuadCLT ref_scene, // now - may be null - for testing if scene is rotated ref
			true, // final boolean show_nan,
			QuadCLT.THREADS_MAX, // int threadsMax,
			debugLevel); // final int debugLevel)
			}
			final float [][] ffclt = fclt;
			ai.set(0);
			for (int ithread = 0; ithread < threads.length; ithread++) {
			threads[ithread] = new Thread() {
			public void run() {
			for (int nTileAll = ai.getAndIncrement(); nTileAll < tiles_td_all; nTileAll = ai.getAndIncrement()) {
			int ntile = nTileAll % tiles_td;
			int nsens = nTileAll / tiles_td;
			int indx0 = ntile * tile_size_td;
			int indx1 = indx0 + tile_size_td;
			for (int indx = indx0; indx < indx1; indx++) {
			float d =ffclt[nsens][indx];
			if (!Float.isNaN(d)){
			sumFclt[nsens][indx] += d;
			numAcc[nsens][indx]++ ;
			}
			}
			}
			}
			};
			}
			ImageDtt.startAndJoin(threads);
			}
			// calculate averages
			ai.set(0);
			for (int ithread = 0; ithread < threads.length; ithread++) {
			threads[ithread] = new Thread() {
			public void run() {
			for (int nTileAll = ai.getAndIncrement(); nTileAll < tiles_td_all; nTileAll = ai.getAndIncrement()) {
			int ntile = nTileAll % tiles_td;
			int nsens = nTileAll / tiles_td;
			int indx0 = ntile * tile_size_td;
			int indx1 = indx0 + tile_size_td;
			for (int indx = indx0; indx < indx1; indx++) {
			if (numAcc[nsens][indx] > 0) {
			sumFclt[nsens][indx]/=numAcc[nsens][indx];
			} else {
			sumFclt[nsens][indx] = Float.NaN;
			}
			}
			}
			}
			};
			}
			ImageDtt.startAndJoin(threads);
			return sumFclt;
			}


	public static double [][] getSceneSZXY(		public static double [][] getSceneSZXY(

src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java

+497 −2

File changed.

Preview size limit exceeded, changes collapsed.

Admin message