Reimplemented/tested pairwise GPU correlation (411beb9e) · Commits · Elphel / imagej-elphel

src/main/java/com/elphel/imagej/gpu/GpuQuad.java

+22 −5

Original line number	Original line	Diff line number	Diff line
	@@ -4473,12 +4473,29 @@ public class GpuQuad{ // quad camera description
	public static TpTask[][] setRectilinearInterTasks(		public static TpTask[][] setRectilinearInterTasks(
	final float [][] fpixels, // to check for empty		final float [][] fpixels, // to check for empty
	final int img_width,		final int img_width,
			Rectangle woi,
	final double [][][] affine // [2][2][3] affine coefficients to translate common to 2 images		final double [][][] affine // [2][2][3] affine coefficients to translate common to 2 images
	){		){
	final int img_height = fpixels[0].length/img_width;		final int img_height = fpixels[0].length/img_width;
	final int tilesX = img_width / GPUTileProcessor.DTT_SIZE;		if (woi == null) {
	final int tilesY = img_height / GPUTileProcessor.DTT_SIZE;		woi = new Rectangle(0,0,img_width,img_height);
	final int tiles = tilesX*tilesY;		} else {
			int x0 = ((int) Math.floor(1.0woi.x/GPUTileProcessor.DTT_SIZE))GPUTileProcessor.DTT_SIZE;
			int y0 = ((int) Math.floor(1.0woi.y/GPUTileProcessor.DTT_SIZE))GPUTileProcessor.DTT_SIZE;
			woi.width += woi.x-x0;
			woi.height += woi.y-y0;
			woi.width = ((int) Math.ceil(1.0woi.width / GPUTileProcessor.DTT_SIZE)) GPUTileProcessor.DTT_SIZE;
			woi.height = ((int) Math.ceil(1.0woi.height/ GPUTileProcessor.DTT_SIZE)) GPUTileProcessor.DTT_SIZE;
			woi.x = x0;
			woi.y = y0;
			}
			final Rectangle tiles_woi = new Rectangle (
			woi.x / GPUTileProcessor.DTT_SIZE,
			woi.y / GPUTileProcessor.DTT_SIZE,
			woi.width / GPUTileProcessor.DTT_SIZE,
			woi.height / GPUTileProcessor.DTT_SIZE);
			final int tiles = tiles_woi.width*tiles_woi.height;

	final int task_code = (1 << GPUTileProcessor.TASK_CORR_EN) \| (1 << GPUTileProcessor.TASK_INTER_EN);		final int task_code = (1 << GPUTileProcessor.TASK_CORR_EN) \| (1 << GPUTileProcessor.TASK_INTER_EN);
	final TpTask[][] tp_tasks_full = new TpTask[2][tiles]; // sparse, rfirst index - image #		final TpTask[][] tp_tasks_full = new TpTask[2][tiles]; // sparse, rfirst index - image #
	final Thread[] threads = ImageDtt.newThreadArray();		final Thread[] threads = ImageDtt.newThreadArray();
	@@ -4492,8 +4509,8 @@ public class GpuQuad{ // quad camera description
	double [][] cxy = new double [2][2]; // image number, {x,y}		double [][] cxy = new double [2][2]; // image number, {x,y}
	int [][] icxy = new int [2][2];		int [][] icxy = new int [2][2];
	for (int nTile = ai.getAndIncrement(); nTile < tiles; nTile = ai.getAndIncrement()) {		for (int nTile = ai.getAndIncrement(); nTile < tiles; nTile = ai.getAndIncrement()) {
	int tileY = nTile / tilesX;		int tileY = nTile / tiles_woi.width + tiles_woi.x;
	int tileX = nTile % tilesX;		int tileX = nTile % tiles_woi.width + tiles_woi.y;
	double [] cxy0 = {		double [] cxy0 = {
	(tileX + 0.5) * GPUTileProcessor.DTT_SIZE,		(tileX + 0.5) * GPUTileProcessor.DTT_SIZE,
	(tileY + 0.5) * GPUTileProcessor.DTT_SIZE};		(tileY + 0.5) * GPUTileProcessor.DTT_SIZE};

src/main/java/com/elphel/imagej/orthomosaic/ComboMatch.java

+66 −13

Original line number	Original line	Diff line number	Diff line
	package com.elphel.imagej.orthomosaic;		package com.elphel.imagej.orthomosaic;

			import java.awt.Rectangle;
	import java.io.IOException;		import java.io.IOException;
	import java.time.LocalDateTime;		import java.time.LocalDateTime;
	import java.util.Arrays;		import java.util.Arrays;
	@@ -142,17 +143,32 @@ public class ComboMatch {
	imp_alt.show();		imp_alt.show();
	}		}

	float [][] gpu_pair_img = maps_collection.getPaddedPairGPU(		System.out.println("Setting up GPU");
	gpu_spair, // String [] spair,		if (GPU_QUAD_AFFINE == null) {
	zoom_lev); // int zoom_lev);		try {
			GPU_QUAD_AFFINE = new GpuQuad(//
			GPU_TILE_PROCESSOR, // GPUTileProcessor gpuTileProcessor,
			gpu_max_width, // final int max_width,
			gpu_max_height, // final int max_height,
			1, // final int num_colors, // normally 1?
			clt_parameters.gpu_debug_level);
			} catch (Exception e) {
			System.out.println("Failed to initialize GpuQuad class");
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
			} // final int debugLevel);
			}

	ShowDoubleFloatArrays.showArrays(		double [][] affine0 = {{1,0,0},{0,1,0}}; // will always stay the same
	gpu_pair_img,		double [][] affine1 = {{1,0,0},{0,1,0}}; // will be variable // here in meters, relative to vertical point
	gpu_width,		double [][][] affines = {affine0,affine1};
	gpu_height,		double [][] corr_pair_rslt = maps_collection.correlateOrthoPair(
	true,		clt_parameters, // CLTParameters clt_parameters,
	"gpu_pair-zoom"+zoom_lev+"-"+gpu_spair[0]+"-"+gpu_spair[1],		gpu_spair, // String [] gpu_spair,
	gpu_spair);		affines, // double [][][] affines, // on top of GPS offsets
			zoom_lev, // int zoom_lev,
			debugLevel); // final int debugLevel)
	if (save_collection) {		if (save_collection) {
	try {		try {
	maps_collection.writeOrthoMapsCollection(orthoMapsCollection_path);		maps_collection.writeOrthoMapsCollection(orthoMapsCollection_path);
	@@ -166,6 +182,28 @@ public class ComboMatch {
	if (debugLevel <1000) {		if (debugLevel <1000) {
	return true;		return true;
	}		}
			/*
			double [] offset_xy_second = {0,0};
			double [][] corr_pair_rslt = maps_collection. correlateOrthoPair(
			clt_parameters, // CLTParameters clt_parameters,
			gpu_spair[0], // String first_name,
			gpu_spair[1], // String second_name,
			zoom_lev, // int zoom_lev,
			offset_xy_second, //double [] offset_xy_second, // on top of affine and GPS
			debugLevel); // final int debugLevel)
			float [][] gpu_pair_img = maps_collection.getPaddedPairGPU(
			gpu_spair, // String [] spair,
			zoom_lev); // int zoom_lev);

			ShowDoubleFloatArrays.showArrays(
			gpu_pair_img,
			gpu_width,
			gpu_height,
			true,
			"gpu_pair-zoom"+zoom_lev+"-"+gpu_spair[0]+"-"+gpu_spair[1],
			gpu_spair);
			*/

	/* */		/* */
	OrthoMap [] ortho_maps = maps_collection.getMaps(); // just temporarily		OrthoMap [] ortho_maps = maps_collection.getMaps(); // just temporarily

	@@ -322,10 +360,12 @@ public class ComboMatch {
	affine[1][1][2] = -0.5pix_shift[1]zoom_scale;		affine[1][1][2] = -0.5pix_shift[1]zoom_scale;


	TDCorrTile [] td_corr_tiles = rectilinearCorrelate_TD( // scene0/scene1		// TDCorrTile [] td_corr_tiles =
			rectilinearVectorField( // rectilinearCorrelate_TD( // scene0/scene1
	clt_parameters, // final CLTParameters clt_parameters,		clt_parameters, // final CLTParameters clt_parameters,
	gpu_pix, // final float [][] fpixels, // to check for empty		gpu_pix, // final float [][] fpixels, // to check for empty
	gpu_width, // final int img_width,		gpu_width, // final int img_width,
			null, // Rectangle woi, // if null, use full GPU window
	affine, // final double [][][] affine, // [2][2][3] affine coefficients to translate common to 2 images		affine, // final double [][][] affine, // [2][2][3] affine coefficients to translate common to 2 images
	false, // final boolean batch_mode,		false, // final boolean batch_mode,
	debugLevel); // final int debugLevel);		debugLevel); // final int debugLevel);
	@@ -337,11 +377,23 @@ public class ComboMatch {

	return true;		return true;
	}		}
			/*
	public static TDCorrTile [] rectilinearCorrelate_TD( // scene0/scene1		public static TDCorrTile [] rectilinearCorrelate_TD( // scene0/scene1
	final CLTParameters clt_parameters,		final CLTParameters clt_parameters,
	final float [][] fpixels, // to check for empty		final float [][] fpixels, // to check for empty
	final int img_width,		final int img_width,
			Rectangle woi, // if null, use full GPU window
			final double [][][] affine, // [2][2][3] affine coefficients to translate common to 2 images
			final boolean batch_mode,
			final int debugLevel) {
			*/


			public static double [][][] rectilinearVectorField( // scene0/scene1
			final CLTParameters clt_parameters,
			final float [][] fpixels, // to check for empty
			final int img_width,
			Rectangle woi, // if null, use full GPU window
	final double [][][] affine, // [2][2][3] affine coefficients to translate common to 2 images		final double [][][] affine, // [2][2][3] affine coefficients to translate common to 2 images
	final boolean batch_mode,		final boolean batch_mode,
	final int debugLevel) {		final int debugLevel) {
	@@ -349,6 +401,7 @@ public class ComboMatch {
	TpTask [][] tp_tasks = GpuQuad.setRectilinearInterTasks(		TpTask [][] tp_tasks = GpuQuad.setRectilinearInterTasks(
	fpixels, // final float [][] fpixels, // to check for empty		fpixels, // final float [][] fpixels, // to check for empty
	img_width, // final int img_width,		img_width, // final int img_width,
			woi, // Rectangle woi,
	affine); // final double [][][] affine // [2][2][3] affine coefficients to translate common to 2 images		affine); // final double [][][] affine // [2][2][3] affine coefficients to translate common to 2 images
	boolean is_aux = true;		boolean is_aux = true;
	boolean is_mono = true;		boolean is_mono = true;
	@@ -509,7 +562,7 @@ public class ComboMatch {
	}		}


	return corr_tiles;		return vector_field; // corr_tiles;
	}		}

src/main/java/com/elphel/imagej/orthomosaic/OrthoMap.java

+17 −1

Original line number	Original line	Diff line number	Diff line
	@@ -306,11 +306,24 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
	}		}

	/**		/**
	* Get metric bounds of this image (zero point at vert_meters)		* Get metric bounds of this image (zero point at vert_meters) with the affine transform
			* stored with this image.
	* @param rectified if true, use rectified (inverse-transformed) image, false - original		* @param rectified if true, use rectified (inverse-transformed) image, false - original
	* @return rectified {{x_min, x_max},{y_min,y_max}}		* @return rectified {{x_min, x_max},{y_min,y_max}}
	*/		*/
	public double [][] getBoundsMeters (boolean rectified){		public double [][] getBoundsMeters (boolean rectified){
			return getBoundsMeters (rectified, affine);
			}

			/**
			* Get metric bounds of this image (zero point at vert_meters) with specified affine transform
			* @param rectified if true, use rectified (inverse-transformed) image, false - original
			* @param affine - 6-element affine transform
			* @return rectified {{x_min, x_max},{y_min,y_max}}
			*/
			public double [][] getBoundsMeters (
			boolean rectified,
			double [][] affine){
	double [][] corners = get4SourceCornersMeters();		double [][] corners = get4SourceCornersMeters();
	if (rectified) {		if (rectified) {
	double [][] inv_aff = invertAffine(affine);		double [][] inv_aff = invertAffine(affine);
	@@ -333,6 +346,9 @@ public class OrthoMap implements Comparable <OrthoMap>, Serializable{
	return bounds;		return bounds;
	}		}




	/**		/**
	* Get pixel bounds of this image (zero point at vert_meters) as doubles (to be able to		* Get pixel bounds of this image (zero point at vert_meters) as doubles (to be able to
	* offset before converting to int.		* offset before converting to int.

src/main/java/com/elphel/imagej/orthomosaic/OrthoMapsCollection.java

+200 −0

Original line number	Original line	Diff line number	Diff line
	package com.elphel.imagej.orthomosaic;		package com.elphel.imagej.orthomosaic;

			import java.awt.Rectangle;
	import java.io.File;		import java.io.File;
	import java.io.FileInputStream;		import java.io.FileInputStream;
	import java.io.FileOutputStream;		import java.io.FileOutputStream;
	@@ -20,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger;
	import com.elphel.imagej.cameras.CLTParameters;		import com.elphel.imagej.cameras.CLTParameters;
	import com.elphel.imagej.common.ShowDoubleFloatArrays;		import com.elphel.imagej.common.ShowDoubleFloatArrays;
	import com.elphel.imagej.tileprocessor.ImageDtt;		import com.elphel.imagej.tileprocessor.ImageDtt;
			import com.elphel.imagej.tileprocessor.TDCorrTile;

	import ij.ImagePlus;		import ij.ImagePlus;
	import ij.ImageStack;		import ij.ImageStack;
	@@ -173,6 +175,38 @@ public class OrthoMapsCollection implements Serializable{
	return bounds;		return bounds;
	}		}

			/**
			* Get rectified overlap bounds of two provided ortho images relative to the origin (vertical
			* point) of the first one in meters. Use specified affine transforms, not saved with the orto map
			* @param ref_index - index of the first (reference) map
			* @param other_index - index of the second (other) map
			* @param ref_affine - specified affine transform of the reference image (referenced to its vertical_point)
			* @param other_affine - specified affine transform of the other image (referenced to its vertical_point)
			* @return {{min_x,max_x},{min_y,max_y}} bounds that include an overlap of the two provided maps
			* relative to the origin (vertical point) of the first image.
			* Returns null if there is no overlap
			*/
			private double [][] getOverlapMeters(
			int ref_index,
			int other_index,
			double [][] ref_affine,
			double [][] other_affine){
			double [][] bounds = ortho_maps[ref_index].getBoundsMeters(true,ref_affine);
			double [][] bounds_other = ortho_maps[other_index].getBoundsMeters(true,other_affine);
			double [] enuOffset = ortho_maps[ref_index].enuOffsetTo(ortho_maps[other_index]);
			double [] rd = {enuOffset[0], -enuOffset[1]}; // {right,down} of the image
			for (int n = 0; n < bounds.length; n++) {
			bounds[n][0] = Math.max(bounds[n][0],bounds_other[n][0]+ rd[n]);
			bounds[n][1] = Math.min(bounds[n][1],bounds_other[n][1]+ rd[n]);
			if (bounds[n][0] > bounds[n][1]) {
			return null;
			}
			}
			return bounds;
			}



	public ImagePlus renderMulti (		public ImagePlus renderMulti (
	String title,		String title,
	boolean use_alt,		boolean use_alt,
	@@ -336,6 +370,7 @@ public class OrthoMapsCollection implements Serializable{
	return gpu_pair_img;		return gpu_pair_img;
	}		}

			/*
	public double [][] correlateOrthoPair(		public double [][] correlateOrthoPair(
	CLTParameters clt_parameters,		CLTParameters clt_parameters,
	String first_name,		String first_name,
	@@ -379,7 +414,172 @@ public class OrthoMapsCollection implements Serializable{
	map_names);		map_names);
	}		}
	return null;		return null;
			} */


			public double [][] correlateOrthoPair(
			CLTParameters clt_parameters,
			String [] gpu_spair,
			double [][][] affines, // here in meters, relative to vertical points
			int zoom_lev,
			final int debugLevel){
			int [] gpu_pair = new int[gpu_spair.length];
			for (int i = 0; i < gpu_pair.length; i++) {
			gpu_pair[i] = map_index_string.get(gpu_spair[i]);
			}
			return correlateOrthoPair(
			clt_parameters, // CLTParameters clt_parameters,
			gpu_pair, // int [] gpu_pair,
			affines, // double [][][] affines, // here in meters, relative to vertical points
			zoom_lev, // int zoom_lev,
			debugLevel); // final int debugLevel)
			}

			private double [][] correlateOrthoPair(
			CLTParameters clt_parameters,
			int [] gpu_pair,
			double [][][] affines, // here in meters, relative to vertical points
			int zoom_lev,
			final int debugLevel){
			double [][] bounds_overlap_meters = getOverlapMeters(
			gpu_pair[0], // int ref_index,
			gpu_pair[1], // int other_index)
			affines[0], // double [][] ref_affine,
			affines[1]); // double [][] other_affine

			if (bounds_overlap_meters == null) {
			if (debugLevel > -3) {
			System.out.println("correlateOrthoPair(): no overlap");
			return null;
			}
			}
			double pix_size = OrthoMap.getPixelSizeMeters (zoom_lev);
			int [] overlap_wh_pixel = new int [2];
			for (int i = 0; i < 2; i++) {
			overlap_wh_pixel[i] = ((int) Math.ceil(bounds_overlap_meters[i][1]/pix_size)) - ((int) Math.floor(bounds_overlap_meters[i][0]/pix_size));
			}
			// double [][] bounds_overlap_pixels = new double[2][2];
			// convert to pixels,shift top-left to [0,0] (remember offsets, limit w,h,
			// change to pixels last, remember TL in meters?
			// keep center where it was
			// {bounds_overlap_meters[0][0],bounds_overlap_meters[1][0],
			double [] enuOffset = ortho_maps[gpu_pair[0]].enuOffsetTo(ortho_maps[gpu_pair[1]]);
			double [] rd = {enuOffset[0], -enuOffset[1]}; // {right,down} of the image
			double [][] tlo_rect_metric = new double [2][2]; // top-left of overlap referenced to it's own vertical point (subtract!)
			tlo_rect_metric[0][0] = bounds_overlap_meters[0][0]; // relative to ref vert_meters
			tlo_rect_metric[0][1] = bounds_overlap_meters[1][0]; // vert_meters
			tlo_rect_metric[1][0] = bounds_overlap_meters[0][0] // relative to other vert_meters
			- rd[0] + ortho_maps[gpu_pair[1]].vert_meters[0]- ortho_maps[gpu_pair[0]].vert_meters[0];
			tlo_rect_metric[1][1] = bounds_overlap_meters[1][0]
			- rd[1] + ortho_maps[gpu_pair[1]].vert_meters[1]- ortho_maps[gpu_pair[0]].vert_meters[1];

			double [][] tlo_src_metric = new double[tlo_rect_metric.length][2]; // relative to it's own vert_meters
			for (int n=0; n <tlo_src_metric.length; n++) {
			for (int i = 0; i < 2; i++) { // subtracting tl_rect_metric[n] (-1)
			tlo_src_metric[n][i] =
			tlo_rect_metric[n][0] * affines[n][i][0] +
			tlo_rect_metric[n][1] * affines[n][i][1] + affines[n][i][2]; // -
			// tl_rect_metric[n][i];
			}
	}		}
			/// referenced to top-left pixel of the gpu image
			double [][] tlo_source_pixel = new double[tlo_src_metric.length][2];
			for (int n=0; n <tlo_source_pixel.length; n++) {
			for (int i = 0; i < 2; i++) {
			tlo_source_pixel[n][i] = (tlo_src_metric[n][i] + ortho_maps[gpu_pair[n]].vert_meters[i])/pix_size;
			}
			}

			double [][][] affines_gpu = new double [affines.length][2][3]; // relative to top left corners, in pixels
			float [][] gpu_pair_img = new float [2][];
			for (int n = 0; n < gpu_pair.length; n++) {
			for (int i = 0; i < 2; i++) {
			for (int j = 0; j < 2; j++) {
			affines_gpu[n][i][j] = affines[n][i][j];
			}
			affines_gpu[n][i][2] = tlo_source_pixel[n][i];
			}
			gpu_pair_img[n] = ortho_maps[gpu_pair[n]].getPaddedGPU (zoom_lev); // int zoom_level,
			}
			Rectangle woi = new Rectangle(0, 0, overlap_wh_pixel[0], overlap_wh_pixel[1]);
			if (woi.width > OrthoMap.gpu_width) {
			if (debugLevel > -3) {
			System.out.println("correlateOrthoPair() correlation woi.width="+woi.width+" > gpu_width="+OrthoMap.gpu_width+". Truncating.");
			}
			woi.width = OrthoMap.gpu_width;
			}
			if (woi.height > OrthoMap.gpu_height) {
			if (debugLevel > -3) {
			System.out.println("correlateOrthoPair() correlation woi.height="+woi.height+" > gpu_height="+OrthoMap.gpu_height+". Truncating.");
			}
			woi.height = OrthoMap.gpu_height;
			}
			final int gpu_width = OrthoMap.gpu_width; // static
			// uses fixed_size gpu image size
			// TDCorrTile [] td_corr_tiles =
			double [][][] vector_field =
			ComboMatch.rectilinearVectorField(//rectilinearCorrelate_TD( // scene0/scene1
			clt_parameters, // final CLTParameters clt_parameters,
			gpu_pair_img, // final float [][] fpixels, // to check for empty
			gpu_width, // final int img_width,
			woi, // Rectangle woi, // if null, use full GPU window
			affines_gpu, // final double [][][] affine, // [2][2][3] affine coefficients to translate common to 2 images
			false, // final boolean batch_mode,
			debugLevel); // final int debugLevel);
			// may use tl_rect_metric to remap to the original image

			/*
			if (show_vector_field) {
			double [][] dbg_vf = new double [3 * vector_field.length][tilesX * tilesY];
			String [] dbg_titles = new String[dbg_vf.length];
			String [] prefix= {"single","neibs"};
			for (int n = 0; n < vector_field.length; n++) {
			dbg_titles [3*n+0] = prefix[n]+"-vx";
			dbg_titles [3*n+1] = prefix[n]+"-vy";
			dbg_titles [3*n+2] = prefix[n]+"-str";
			}
			for (int i = 0; i < dbg_vf.length; i++) {
			Arrays.fill(dbg_vf[i], Double.NaN);

			}
			for (int t=0; t<dbg_vf[0].length; t++) {
			for (int n = 0; n < vector_field.length; n++) {
			if (vector_field[n][t] != null) {
			for (int k = 0; k < 3; k++) {
			dbg_vf[n*3 + k][t] = vector_field[n][t][k];
			}
			}
			}
			}
			ShowDoubleFloatArrays.showArrays(
			dbg_vf,
			tilesX,
			tilesY,
			true,
			"vector_field",
			dbg_titles);
			}

			*/
			if (debugLevel > 1) {
			String [] map_names = {ortho_maps[gpu_pair[0]].getName(),ortho_maps[gpu_pair[1]].getName()};
			ShowDoubleFloatArrays.showArrays(
			gpu_pair_img,
			OrthoMap.gpu_width,
			OrthoMap.gpu_height,
			true,
			"gpu_pair-zoom"+zoom_lev+"-"+ortho_maps[gpu_pair[0]].getName()+"-"+ortho_maps[gpu_pair[1]].getName(),
			map_names);
			}
			return null;
			}








	public static OrthoMapsCollection readOrthoMapsCollection (		public static OrthoMapsCollection readOrthoMapsCollection (
	String path) throws IOException, ClassNotFoundException {		String path) throws IOException, ClassNotFoundException {

Admin message