Working on occluded BG. Some works

src/main/java/com/elphel/imagej/gpu/TpTask.java

@@ -176,6 +176,10 @@ public class TpTask {
 	public int getTileX(){
 		return tx;
 	}
+	public int [] getTileXY(){
+		return new int [] {tx,ty};
+	}
+	
 	public int getTask() {
 		return task;
 	}

src/main/java/com/elphel/imagej/tileprocessor/QuadCLT.java

@@ -1775,24 +1775,25 @@ public class QuadCLT extends QuadCLTCPU {
 	}
 	
 	public static double [][][][] texturesNoOverlapGPUFromDSI(
-			CLTParameters     clt_parameters,
-			double []         disparity_ref,
+			CLTParameters      clt_parameters,
+			double []          disparity_ref,
 			// motion blur compensation 
-			double            mb_tau,      // 0.008; // time constant, sec
-			double            mb_max_gain, // 5.0;   // motion blur maximal gain (if more - move second point more than a pixel
-			double [][]       mb_vectors,  // now [2][ntiles];
-			final double []   scene_xyz, // camera center in world coordinates
-			final double []   scene_atr, // camera orientation relative to world frame
-			final QuadCLT     scene,
-			final QuadCLT     ref_scene, // now - may be null - for testing if scene is rotated ref
-			final boolean     filter_bg, // remove bg tiles (possibly occluded)
-			final double      max_distortion, // maximal neighbor tiles offset as a fraction of tile size (8) 
-			final int []      cluster_index,  // [tilesX*tilesY]
-			final boolean []  border, // border tiles for clusters?
-			final int         discard_frame_edges, // do not use tiles that have pixels closer to the frame margins 
-			final int         keep_frame_tiles, // do not discard pixels for border tiles in reference frame 
-			final boolean     keep_channels,
-			final int         debugLevel){
+			double             mb_tau,      // 0.008; // time constant, sec
+			double             mb_max_gain, // 5.0;   // motion blur maximal gain (if more - move second point more than a pixel
+			double [][]        mb_vectors,  // now [2][ntiles];
+			final double []    scene_xyz, // camera center in world coordinates
+			final double []    scene_atr, // camera orientation relative to world frame
+			final QuadCLT      scene,
+			final QuadCLT      ref_scene, // now - may be null - for testing if scene is rotated ref
+			final boolean      filter_bg, // remove bg tiles (possibly occluded)
+			final double       max_distortion, // maximal neighbor tiles offset as a fraction of tile size (8) 
+			final int []       cluster_index,  // [tilesX*tilesY]
+			final boolean []   border, // border tiles for clusters?
+			final int          discard_frame_edges, // do not use tiles that have pixels closer to the frame margins 
+			final int          keep_frame_tiles, // do not discard pixels for border tiles in reference frame 
+			final boolean      keep_channels,
+			final TpTask[][][] tp_tasks_ret, // if not null, should be [1] - will return tp_tasks_ret[0] = tp_tasks
+			final int          debugLevel){
 		// FIXME: Move to clt_parameters;
 		final double max_overlap =    0.6; 
 		final double min_adisp_cam =  0.2;
@@ -1899,11 +1900,12 @@ public class QuadCLT extends QuadCLTCPU {
 		if (tp_tasks[0].length == 0) {
 			if (debugLevel > -1) {
 				System.out.println("texturesGPUFromDSI(): no tiles to process");
-				
 			}
 			return null;
 		}
-		
+		if (tp_tasks_ret != null) {
+			tp_tasks_ret[0] = tp_tasks; // return tp_tasks to caller
+		}
 		///		scene.saveQuadClt(); // to re-load new set of Bayer images to the GPU (do nothing for CPU) and Geometry
 		ImageDtt image_dtt = new ImageDtt(
 				scene.getNumSensors(),

src/main/java/com/elphel/imagej/tileprocessor/TexturedModel.java

@@ -38,6 +38,7 @@ import com.elphel.imagej.cameras.EyesisCorrectionParameters;
 import com.elphel.imagej.common.DoubleGaussianBlur;
 import com.elphel.imagej.common.ShowDoubleFloatArrays;
 import com.elphel.imagej.correction.EyesisCorrections;
+import com.elphel.imagej.gpu.TpTask;
 import com.elphel.imagej.x3d.export.GlTfExport;
 import com.elphel.imagej.x3d.export.TriMesh;
 import com.elphel.imagej.x3d.export.WavefrontExport;
@@ -2999,6 +3000,38 @@ public class TexturedModel {
 		return rslt;
 	}
 	
+	public static double [][][][] getPixelOffsets(
+		final TpTask[][][]   tp_tasks_ref, //
+		final boolean [][][] tile_booleans, // to filter?
+		final int         tilesX)
+	{
+		final int num_slices = tile_booleans[0].length;
+		final int num_tiles = tile_booleans[0][0].length;
+		double [][][][] pix_offsets = new double [num_slices][num_tiles][][];
+		final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
+		final AtomicInteger ai = new AtomicInteger(0);
+		ai.set(0);
+		for (int ithread = 0; ithread < threads.length; ithread++) {
+			threads[ithread] = new Thread() {
+				public void run() {
+					for (int nslice = ai.getAndIncrement(); nslice < num_slices; nslice = ai.getAndIncrement()) {
+						if ((tp_tasks_ref[nslice]!= null) && (tp_tasks_ref[nslice].length>0) &&  (tp_tasks_ref[nslice][0]!= null)) {
+							for (int ntile = 0; ntile <  tp_tasks_ref[nslice][0].length; ntile++) {
+								TpTask task = tp_tasks_ref[nslice][0][ntile];
+								int tile = task.getTileX()+task.getTileY()*tilesX;
+								pix_offsets[nslice][tile] = task.getDoubleXY();
+							}
+						}
+					}
+				}
+			};
+		}		      
+		ImageDtt.startAndJoin(threads);
+
+		return pix_offsets;
+	}
+	
+	
 	/**
 	 * Select pixels between weak tiles and strong tiles for both has_bg (edge where
 	 * triangular mesh will end) and is_fg tiles extending 4 pixels over weak foreground
@@ -4578,9 +4611,233 @@ public class TexturedModel {
 		}		      
 		ImageDtt.startAndJoin(threads);
 		return;
-	}	
+	}
+	
+	/**
+	 * Generate bitmask of sensors that should be removed from the composite texture. Uses
+	 * image offsets from TileTask array to get shift between textures rendered for different
+	 * disparities. The source (pre-aberration) offsets directly are not used, just difference
+	 * for the same sensors.
+	 * Considering for being occluded all but strong FG tiles
+	 *  
+	 * @param channel_pixel_offsets per-slice, per tile (linescan order), per-sensor x,y offsets.
+	 * @param alpha_pix         boolean "alpha" - true - opaque, false - transparent
+	 * @param slice_disparities per-tile disparities ([slice][tile]).
+	 * @param tile_keep         boolean map of kept tiles (tile_booleans[TILE_KEEP])
+	 * @param tile_fg_strong    boolean map of strong FG tiles (tile_booleans[TILE_IS_FG_STRONG])
+	 * @param tile_stitch       boolean map of stitch tiles (tile_booleans[TILE_STITCH]) - they have duplicates
+	 * @param occlusion_frac    thershold for interpolating occlusion - fraction of BG tile being occluded
+	 *                          to actually occlude
+	 * @param width             image width in pixels 
+	 * @param transform_size    CLT conversion size. Always 8
+	 * @return                  [nslice][pix] bit map of occluded sensors to be removed from sources of the
+	 *                          combined textures. 
+	 */
+	public static int [][] getOccludedMap(
+			final double [][][][] channel_pixel_offsets,
+			final boolean [][]    alpha_pix,
+			final double  [][]    slice_disparities,
+			final boolean [][]    tile_keep,      // do not check occluded strong foreground
+			final boolean [][]    tile_fg_strong, // do not check occluded strong foreground
+			final boolean [][]    tile_stitch,    // do not process these - there are duplicates
+			final double          occlusion_frac, // ratio of opaque pixel overlap to consider occlusion 
+			final int             width,
+			final int             transform_size){
+		final int num_slices = alpha_pix.length;
+		final int img_size =   alpha_pix[0].length;
+		final int height = img_size/width;
+		final int tilesX = width/transform_size;
+		final int tilesY = img_size/width/transform_size;
+		final int tiles =  tilesX * tilesY;
+		final int dbg_tile = 4123;
+		final int dbg_slice = 0;
+		final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
+		final AtomicInteger ai = new AtomicInteger(0);
+		final int [][] occluded = new int [num_slices][img_size];
+		for (int nslice = 0; nslice < num_slices; nslice++) {
+			int fnslice = nslice;
+			ai.set(0);
+			for (int ithread = 0; ithread < threads.length; ithread++) {
+				threads[ithread] = new Thread() {
+					public void run() {
+						for (int tile = ai.getAndIncrement(); tile < tiles; tile = ai.getAndIncrement()) {
+							if ((fnslice == dbg_slice) && (tile == dbg_tile )) {
+								System.out.println("getNonOccludedMap().1 nslice="+fnslice+", tile="+tile);
+							}
+							double [][] offs_bg = channel_pixel_offsets[fnslice][tile];
+							if (tile_keep[fnslice][tile] && !tile_fg_strong[fnslice][tile] && !tile_stitch[fnslice][tile]) {
+								for (int ns = 0; ns < num_slices; ns++) {
+									if ((ns != fnslice) &&
+											tile_keep[ns][tile] &&
+											!tile_stitch[fnslice][tile] &&
+											(slice_disparities[ns][tile] > slice_disparities[fnslice][tile])) {
+										double [][] offs_fg = channel_pixel_offsets[ns][tile];
+										double [][] pixel_offs = new double [offs_bg.length][2];
+										for (int nsens = 0; nsens < pixel_offs.length; nsens++) {
+											if (offs_bg[nsens] != null) { // to implement sensor mask later
+												pixel_offs[nsens][0] = offs_bg[nsens][0] - offs_fg[nsens][0];
+												pixel_offs[nsens][1] = offs_bg[nsens][1] - offs_fg[nsens][1];
+											}
+										}
+										int tileX = tile % tilesX; 
+										int tileY = tile / tilesX;
+										for (int dy = 0; dy < transform_size; dy++) {
+											int py0 = tileY * transform_size + dy;
+											for (int dx = 0; dx < transform_size; dx++) {
+												int px0 = tileX * transform_size + dx;
+												int occluded_mask = 0;
+												for (int nsens = 0; nsens < pixel_offs.length; nsens++) if (offs_bg[nsens] != null) {
+													double px = px0 + pixel_offs[nsens][0]; 
+													double py = py0 + pixel_offs[nsens][1];
+													if ((px >= 0) && (px < (width - 1)) && (py >= 0) && (py < (height - 1))) {
+														int ipx = (int) Math.floor(px);
+														int ipy = (int) Math.floor(py);
+														int indx_fg = ipx + ipy*width;
+														boolean occl_any = 
+																alpha_pix[ns][indx_fg] ||
+																alpha_pix[ns][indx_fg + 1] ||
+																alpha_pix[ns][indx_fg + width] ||
+																alpha_pix[ns][indx_fg + width + 1];
+														boolean occl_all = 
+																alpha_pix[ns][indx_fg] &&
+																alpha_pix[ns][indx_fg + 1] &&
+																alpha_pix[ns][indx_fg + width] &&
+																alpha_pix[ns][indx_fg + width + 1];
+														if (occl_all) {
+															occluded_mask |= (1 << nsens);
+														} else {
+															if (occl_any) {
+																double fx =  px - ipx;
+																double fy =  py - ipy;
+																double d = 0;
+																if (alpha_pix[ns][indx_fg])             d += (1 - fx) * (1- fy);
+																if (alpha_pix[ns][indx_fg + 1])         d += (    fx) * (1- fy);
+																if (alpha_pix[ns][indx_fg + width])     d += (1 - fx) * (   fy);
+																if (alpha_pix[ns][indx_fg + width + 1]) d += (    fx) * (   fy);
+																if (d >= occlusion_frac) {
+																	occluded_mask |= (1 << nsens);
+																}
+															}
+														}
+													}
+												} // for (int nsens = 0; nsens < pixel_offs.length; nsens++) {
+												int indx = (((tileY * width + tileX) * transform_size) + dy * width) + dx;
+												occluded[fnslice][indx] |= occluded_mask;
+											}
+										}
+									} // if ((ns != fnslice) && ...
+								}
+							}
+						}
+					}
+				};
+			}		      
+			ImageDtt.startAndJoin(threads);
+		}
+		// duplicate for stitch tiles
+		for (int nslice = 0; nslice < num_slices; nslice++) {
+			int fnslice = nslice;
+			ai.set(0);
+			for (int ithread = 0; ithread < threads.length; ithread++) {
+				threads[ithread] = new Thread() {
+					public void run() {
+						for (int tile = ai.getAndIncrement(); tile < tiles; tile = ai.getAndIncrement()) {
+							if (tile_keep[fnslice][tile] && !tile_fg_strong[fnslice][tile] && tile_stitch[fnslice][tile]) {
+								for (int ns = 0; ns < num_slices; ns++) {
+									if ((ns != fnslice) && // other layer with same disparity and non-stitch (probably stitched) 
+											tile_keep[ns][tile] &&
+											!tile_stitch[fnslice][tile] &&
+											(slice_disparities[ns][tile] == slice_disparities[fnslice][tile])) {
+										int tileX = tile % tilesX; 
+										int tileY = tile / tilesX;
+										for (int dy = 0; dy < transform_size; dy++) {
+											int indx0 = (tileY * transform_size + dy) * width + tileX * transform_size;
+											System.arraycopy(
+													occluded[ns],
+													indx0,
+													occluded[fnslice],
+													indx0,
+													transform_size);
+										}
+									}									
+								}
+							}
+						}
+					}
+				};
+			}		      
+			ImageDtt.startAndJoin(threads);
+		}		
+		return occluded;
+		// for debug - display number of bits from bit_mask
+	}
 	
+	public static double [][] debugOccludedMap(
+			final int     [][]   occluded_map
+			){
+		final int num_slices =  occluded_map.length;
+		final int img_size =    occluded_map[0].length;
+		double [][] dbg_map = new double [num_slices][img_size];
+		for (int nslice = 0; nslice < num_slices; nslice++) {
+			for (int pix = 0; pix < img_size; pix++) {
+				if (occluded_map[nslice][pix] != 0) {
+					int n = 0;
+					for (int d = occluded_map[nslice][pix]; d != 0;  d >>= 1) {
+						if ((d & 1) != 0) {
+							n++;
+						}
+					}
+					dbg_map[nslice][pix] = n;
+				}
+			}
+		}		
+		return dbg_map;
+	}
 	
+	public static double [][] combineTexturesWithOcclusions(
+			final double  [][][] sensor_texture,
+			final double  [][]   combo_texture,
+			final int     [][]   occluded_map){
+		final int num_slices =  sensor_texture.length;
+		final int img_size =    combo_texture[0].length;
+		final int num_sensors = sensor_texture[0].length;
+		final double [][] occluded_texture = new double [num_slices][img_size];
+		final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
+		final AtomicInteger ai = new AtomicInteger(0);
+		for (int nslice = 0; nslice < num_slices; nslice++) {
+			final int fnslice = nslice;
+			ai.set(0);
+			for (int ithread = 0; ithread < threads.length; ithread++) {
+				threads[ithread] = new Thread() {
+					public void run() {
+						for (int pix = ai.getAndIncrement(); pix < img_size; pix = ai.getAndIncrement()) {
+							if (occluded_map[fnslice][pix] == 0) {
+								occluded_texture[fnslice][pix] = combo_texture[fnslice][pix];
+							} else {
+								int num_used_sensors = 0;
+								int msk = occluded_map[fnslice][pix];
+								double s = 0.0;
+								for (int nsens = 0; nsens < num_sensors; nsens++ ) {
+									if ((msk & (1 << nsens)) == 0) {
+										s += sensor_texture[fnslice][nsens][pix];
+										num_used_sensors++;
+									}
+								}
+								if (num_used_sensors > 0) {
+									s /= num_used_sensors;
+								} else {
+									s = Double.NaN;
+								}
+								occluded_texture[fnslice][pix] = s;
+							}
+						}
+					}
+				};
+			}		      
+			ImageDtt.startAndJoin(threads);
+		}
+		return occluded_texture;
+	}
 	
 	
 	
@@ -5030,6 +5287,7 @@ public class TexturedModel {
 			final TileCluster[]  tileClusters, // to process blue_sky?
 			final double         max_disparity_lim, //  = 100.0;  // do not allow stray disparities above this
 			final double         min_trim_disparity, //  =  2.0;  // do not try to trim texture outlines with lower disparities
+			final TpTask[][][]   tp_tasks_ref,       // reference tasks for each slice to get offsets			
 			final String         dbg_prefix) {
 		final double var_radius =         1.5; // 3.5;   // for variance filter of the combo disparity
 		final double dir_radius =         1.5;   // averaging inter-sensor variance to view behind obstacles 
@@ -5095,7 +5353,12 @@ public class TexturedModel {
 				max_neib_lev,                      // final int         max_neib_lev,
 				transform_size,                    // final int         transform_size,
 				tilesX);                           // final int         tilesX)
-		
+
+		double [][][][] channel_pixel_offsets = getPixelOffsets(
+				tp_tasks_ref,  //final TpTask[][][]   tp_tasks_ref, //
+				tile_booleans, //final boolean [][][] tile_booleans, // to filter?
+				tilesX);       // final int         tilesX)
+
 		if (dbg_prefix != null) {
 			double [][] dbg_img = new double [tile_booleans[0].length * 5][tile_booleans[0][0].length];
 			String[] dbg_titles = new String [tile_booleans[0].length * 5];
@@ -5216,6 +5479,7 @@ public class TexturedModel {
 				first_trimmed_alpha[i] = unbound_alpha[i].clone();
 			}
 		}
+		// not used:
 		final boolean       dual_pass = false; // true;
 		expandTrimAlpha(
 				trim_pixels,    // final boolean [][]  trim_pix,  // pixels that may be trimmed
@@ -5304,7 +5568,25 @@ public class TexturedModel {
 				width,                    // final int            width,
 				transform_size);          // final int            transform_size)
 
+// Processing BG	
+		final double          occlusion_frac = 0.9;
+		int [][] occluded_map = getOccludedMap(
+				channel_pixel_offsets,            // final double [][][][] channel_pixel_offsets,
+				unbound_alpha,                    // final boolean [][]    alpha_pix,
+				slice_disparities,                // final double  [][]    slice_disparities,
+				tile_booleans[TILE_KEEP],         // final boolean [][]    tile_keep,      // do not check occluded strong foreground
+				tile_booleans[TILE_IS_FG_STRONG], // final boolean [][]    tile_fg_strong, // do not check occluded strong foreground
+				tile_booleans[TILE_STITCH],       // final boolean [][]    tile_stitch,    // do not process these - there are duplicates
+				occlusion_frac,                   // final double          occlusion_frac, // ratio of opaque pixel overlap to consider occlusion 
+				width,                            // final int             width,
+				transform_size);                  // final int             transform_size);
 		
+		final double  [][] dbg_occluded_map = (dbg_prefix == null)? null:debugOccludedMap(occluded_map);
+		
+		final double  [][] occluded_textures = combineTexturesWithOcclusions(
+				sensor_texture,  // final double  [][][] sensor_texture,
+				gcombo_texture,  // final double  [][]   combo_texture,
+				occluded_map);   // final int     [][]   occluded_map);
 		
 		
 		
@@ -5598,6 +5880,11 @@ public class TexturedModel {
 						fix_bg_pix,
 						fix_same_pix,
 						trim_alpha_pix,
+						
+						dbg_occluded_map[nslice],
+						gcombo_texture[nslice],
+						occluded_textures[nslice],
+						
 						dbg_text_edge[nslice], // dbg_text_edge,
 						dbg_text_en[nslice],
 						dbg_fg_prefiltered[nslice], //
@@ -5605,7 +5892,7 @@ public class TexturedModel {
 						dbg_fg_prefiltered_neibs[nslice],
 						gtext_fg_filt[nslice], //dbg_fg_filtered[nslice],
 						gdbg_is_fg[nslice],
-						gcombo_texture[nslice],
+//						gcombo_texture[nslice],
 						out_textures  [nslice], // dirs_avg,
 						dbg_out_textures[nslice],
 						dbg_out[5][nslice],
@@ -5661,6 +5948,10 @@ public class TexturedModel {
 						"FIX_HAS_BG",
 						"FIX_SAME",
 						"TRIM_ALPHA",
+						
+						"OCCLUSIONS_MAP",
+						"COMBO_TEXTURE",
+						"OCCLUDED_TEXTURES",
 						"TEXTURE_EDGE",
 						"TEXTURE_ON",
 						"TEXTURE_TRIMMED",
@@ -5668,7 +5959,7 @@ public class TexturedModel {
 						"TEXTURE_TRIMMED_EDGED",
 						"TEXTURE_FG_FILTERED",
 						"IS_FG",
-						"COMBO_TEXTURE",
+//						"COMBO_TEXTURE",
 						"OUT_TEXTURE_BG",
 						"OUT_TEXTURE_FG",
 						"TILE_ALPHA",
@@ -5697,6 +5988,7 @@ public class TexturedModel {
 						true,
 						dbg_prefix+"-textures-"+nslice,
 						dbg_titles);
+				assert true;
 			}
 			ShowDoubleFloatArrays.showArrays(
 					out_textures,
@@ -5864,6 +6156,7 @@ public class TexturedModel {
 		
 		final double [][][] sensor_textures = new double [num_slices][num_sensors][];
 		final double [][] combo_textures = new double [num_slices][];
+		final TpTask[][][] tp_tasks_ref = new TpTask [num_slices][][];
 		for (int nscene = earliestScene; nscene < scenes.length; nscene++) if ((scenes_sel == null) || scenes_sel[nscene]){
 			String ts = scenes[nscene].getImageName();
 			double []   scene_xyz = OpticalFlow.ZERO3;
@@ -5907,6 +6200,12 @@ public class TexturedModel {
 					System.out.println("nscene="+nscene+", nslice="+nslice+" will run texturesGPUFromDSI() that needs debug >2");
 					System.out.print("");
 				}
+				if ((debugLevel > -1) && (nscene == ref_index)) { // change to "-2" to activate
+					System.out.println("Processing reference scene");
+					System.out.print("");
+				}
+				final TpTask[][][] tp_tasks_ret = ((nscene == ref_index) && (tp_tasks_ref != null))?
+								new TpTask[1][][] : null;
 				double [][][][] slice_texture88 = QuadCLT.texturesNoOverlapGPUFromDSI(
 						clt_parameters,          // CLTParameters     clt_parameters,
 						disparity_ref,           // double []         disparity_ref,
@@ -5927,7 +6226,11 @@ public class TexturedModel {
 						10,                      // final int         discard_frame_edges, // do not use tiles that have pixels closer to the frame margins 
 						1,                       // final int         keep_frame_tiles, // do not discard pixels for border tiles in reference frame 
 						true,                    // keep_channels,           // final boolean     keep_channels,
-						debugLevel);            // final int         debugLevel);
+						tp_tasks_ret,            // final TpTask[][][] tp_tasks_ret, // if not null, should be [1] - will return tp_tasks_ret[0] = tp_tasks
+						debugLevel);             // final int         debugLevel);
+				if (tp_tasks_ret != null) {
+					tp_tasks_ref[nslice] = 	tp_tasks_ret[0];
+				}
 				if (slice_texture88 != null) { // will just accumulate
 					// Use MB vectors for texture weights				
 					final Thread[] threads = ImageDtt.newThreadArray(THREADS_MAX);
@@ -6069,6 +6372,7 @@ public class TexturedModel {
 				tileClusters,              // final TileCluster[]  tileClusters, // to process blue_sky?
 				max_disparity_lim,         // final double max_disparity_lim,  // do not allow stray disparities above this
 				min_trim_disparity,        // final double min_trim_disparity, // do not try to trim texture outlines with lower disparities
+				tp_tasks_ref,              // final TpTask[][][]   tp_tasks_ref,       // reference tasks for each slice to get offsets			
 				ref_scene.getImageName()); // null); // ref_scene.getImageName()); // final String         dbg_prefix);
 		if (debugLevel > -1) {
 			double [][] dbg_textures = new double [faded_textures.length * faded_textures[0].length][faded_textures[0][0].length];