8-image reports (6c88d9e2) · Commits · Elphel / lwir-nn

nn_eval_lwir-02.py

0 → 100644

+382 −0

Original line number	Original line	Diff line number	Diff line
			#!/usr/bin/env python3

			__copyright__ = "Copyright 2018, Elphel, Inc."
			__license__ = "GPL-3.0+"
			__email__ = "andrey@elphel.com"


			#from PIL import Image

			import os
			import sys
			#import glob

			#import numpy as np

			import imagej_tiffwriter

			import time
			import imagej_tiff as ijt
			import matplotlib.pyplot as plt
			from matplotlib.backends.backend_pdf import PdfPages
			import qcstereo_functions as qsf
			import numpy as np

			#import xml.etree.ElementTree as ET

			qsf.TIME_START = time.time()
			qsf.TIME_LAST = qsf.TIME_START

			#IMG_WIDTH = 20 # 324 # tiles per image row Defined in config
			#IMG_HEIGHT = 15 # 324 # tiles per image row Defined in config

			DEBUG_LEVEL= 1

			try:
			conf_file = sys.argv[1]
			except IndexError:
			print("Configuration path is required as a first argument. Optional second argument specifies root directory for data files")
			exit(1)
			try:
			root_dir = sys.argv[2]
			except IndexError:
			root_dir = os.path.dirname(conf_file)

			try:
			modes = [sys.argv[3]] # train, infer
			except IndexError:
			modes = ['train']


			print ("Configuration file: " + conf_file)
			parameters, dirs, files, dbg_parameters = qsf.parseXmlConfig(conf_file, root_dir)
			"""
			Temporarily for backward compatibility
			"""
			if not "SLOSS_CLIP" in parameters:
			parameters['SLOSS_CLIP'] = 0.5
			print ("Old config, setting SLOSS_CLIP=", parameters['SLOSS_CLIP'])

			"""
			Defined in config file
			"""
			IMG_WIDTH = None # 20 # 324 # tiles per image row Defined in config
			IMG_HEIGHT = None # 15 # 324 # tiles per image row Defined in config
			TILE_SIDE, TILE_LAYERS, TWO_TRAINS, NET_ARCH1, NET_ARCH2 = [None]*5
			ABSOLUTE_DISPARITY,SYM8_SUB, WLOSS_LAMBDA, SLOSS_LAMBDA, SLOSS_CLIP = [None]*5
			SPREAD_CONVERGENCE, INTER_CONVERGENCE, HOR_FLIP, DISP_DIFF_CAP, DISP_DIFF_SLOPE = [None]*5
			CLUSTER_RADIUS,ABSOLUTE_DISPARITY = [None]*2
			FGBG_MODE = 1 # 0 - do not filter by single-plane, 1 - remove split plabnes tiles, 2 - remove split planes and neighbors
			FIGS_EXTENSIONS = ['png','pdf','svg']
			#FIGS_ESXTENSIONS = ['png','pdf','svg']
			EVAL_MODES = ["train","infer"]
			FIGS_SAVESHOW = ['save','show']


			globals().update(parameters)
			try:
			FIGS_EXTENSIONS = globals()['FIGS_ESXTENSIONS'] # fixing typo in configs
			except:
			pass


			#exit(0)


			TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
			FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
			BATCH_SIZE = ([1,2][TWO_TRAINS])21000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches

			SUFFIX=(str(NET_ARCH1)+'-'+str(NET_ARCH2)+
			(["R","A"][ABSOLUTE_DISPARITY]) +
			(["NS","S8"][SYM8_SUB])+
			"WLAM"+str(WLOSS_LAMBDA)+
			"SLAM"+str(SLOSS_LAMBDA)+
			"SCLP"+str(SLOSS_CLIP)+
			(['_nG','_G'][SPREAD_CONVERGENCE])+
			(['_nI','_I'][INTER_CONVERGENCE]) +
			(['_nHF',"_HF"][HOR_FLIP]) +
			('_CP'+str(DISP_DIFF_CAP)) +
			('_S'+str(DISP_DIFF_SLOPE))
			)

			##############################################################################
			cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
			center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
			qsf.prepareFiles(dirs, files, suffix = SUFFIX)

			#import tensorflow.contrib.slim as slim
			#NN_DISP = 0
			#HEUR_DISP = 1
			#GT_DISP = 2
			#GT_CONF = 3
			#NN_NAN = 4
			#HEUR_NAN = 5
			#NN_DIFF = 6
			#HEUR_DIFF = 7
			# Now - more layers




			CONF_MAX = 0.7
			ERR_AMPL = 0.4 # 0.3
			TIGHT_TOP = 0.95
			TIGHT_HPAD = 1.0
			TIGHT_WPAD = 1.0
			FIGSIZE = [8.5,11.0]
			WOI_COLOR = "red"
			X_COLOR = "grey"
			X_NEIBS = False
			TRANSPARENT = True # for export




			#dbg_parameters
			def get_fig_params(disparity_ranges):
			fig_params = []
			for dr in disparity_ranges:
			if dr[-1][0]=='-':
			fig_params.append(None)
			else:
			subs = []
			for s in dr[:-1]:
			mm = s[:2]
			try:
			lims = s[2]
			except IndexError:
			lims = None
			subs.append({'lim_val':mm, 'lim_xy':lims})
			fig_params.append({'name':dr[-1],'ranges':subs})
			return fig_params

			#try:
			#fig_params = get_fig_params(dbg_parameters['disparity_ranges'])
			extra_path = os.path.join(root_dir, dbg_parameters['extra'])
			eo_width = dbg_parameters['eo_params']['width']
			eo_height = dbg_parameters['eo_params']['height']
			eo_woi = dbg_parameters['eo_params']['woi'] # (x,y,width, height)
			eo_disparity_scale = 1.0/dbg_parameters['eo_params']['disparity_scale'] # 14.2
			image_sets = dbg_parameters['extra_paths'] # list of dictionaries


			pass

			#temporary:
			TIFF_ONLY = False # True
			#max_bad = 2.5 # excludes only direct bad
			max_bad = 2.5 #2.5 # 1.5 # excludes only direct bad
			max_diff = 1.5 # 2.0 # 5.0 # maximal max-min difference
			max_target_err = 1.0 # 0.5 # maximal max-min difference
			max_disp = 5.0

			min_strength = 0.18 #ignore tiles below
			min_neibs = 1
			max_log_to_mm = 0.5 # difference between center average and center should be under this fraction of max-min (0 - disables feature)


			#num_bins = 256 # number of histogram bins
			num_bins = 15 # 50 # number of histogram bins
			use_gt_weights = True # False # True
			index_gt = 2
			index_gt_weight = 3
			index_heur_err = 7
			index_nn_err = 6
			index_fgbg_sngl = 10
			index_fgbg_neib = 11

			index_mm = 23 # 8 # max-min
			index_log = 24 # 9
			index_bad = 25 # 10
			index_num_neibs = 26 # 11

			index_fgbg = [index_fgbg_sngl,index_fgbg_neib][X_NEIBS]
			"""
			Debugging high 9-tile variations, removing error for all tiles with lower difference between max and min
			"""
			#min_diff = 0.25 # remove all flat tiles with spread less than this (do not show on heuristic/network disparity errors subplots
			min_diff = 0 # remove all flat tiles with spread less than this




			max_target_err2 = max_target_err * max_target_err
			if not 'show' in FIGS_SAVESHOW:
			plt.ioff()

			#for mode in ['train','infer']:
			#for mode in ['infer']:
			def cross_out(
			plt,
			cross_out_mask):
			height = cross_out_mask.shape[0]
			width = cross_out_mask.shape[1]
			for row in range (height):
			for col in range(width):
			if cross_out_mask[row,col]:
			xdata = [col-0.3, col+0.3]
			ydata = [row-0.3, row+0.3]
			plt.plot(xdata,ydata,color=X_COLOR)
			ydata = [row+0.3, row-0.3]
			plt.plot(xdata,ydata,color=X_COLOR)

			for mode in modes: # ['train']:
			figs = []
			ffiles = [] # no ext
			def setlimsxy(lim_xy):
			if not lim_xy is None:
			plt.xlim(min(lim_xy[:2]),max(lim_xy[:2]))
			plt.ylim(max(lim_xy[2:]),min(lim_xy[2:]))
			cumul_weights = None
			cmap_disp=plt.get_cmap('viridis') # ('cividis')
			cmap_diff=plt.get_cmap('coolwarm') #('seismic') # ('viridis')
			for nfile, img_pars in enumerate(image_sets):
			if not img_pars is None:
			img_file = files['result'][nfile]
			if mode == 'infer':
			img_file = img_file.replace('.npy','-infer.npy')
			print ("Processing image set: "+img_file)

			data,labels = qsf.result_npy_prepare(img_file, ABSOLUTE_DISPARITY, fix_nan=True, insert_deltas=3)
			cross_out_mask = data[...,index_fgbg] < 0.5 #data.shape = (15,20,27)
			# for subindex, rng in enumerate(fpars['ranges']):

			lim_val = img_pars['range'] # rng['lim_val']
			lim_val[0] -= ERR_AMPL
			lim_xy = [-0.5, IMG_WIDTH - 0.5, -0.5, IMG_HEIGHT - 0.5] # rng['lim_xy']

			#start new image page
			fig = plt.figure(figsize=FIGSIZE)
			fig.canvas.set_window_title(img_pars['title'])
			fig.suptitle(img_pars['title'])

			# Create EO DSI image
			# load tiff image

			img_ds_main = ijt.imagej_tiff(os.path.join(extra_path,img_pars['dsi_path'] ))
			ds_main = img_ds_main.image[...,img_pars['dsi_slice']] * eo_disparity_scale
			ds_main = np.maximum(ds_main, lim_val[0])
			ds_main = np.minimum(ds_main, lim_val[1])
			ax_conf=plt.subplot(322)
			ax_conf.set_title("Hi-res camera disparity map")
			plt.imshow(ds_main, vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
			setlimsxy([-0.5, eo_width-0.5, -0.5, eo_height - 0.5])
			if not eo_woi is None:
			pass # show frame
			xdata=[eo_woi['x'], eo_woi['x'] + eo_woi['width'], eo_woi['x'] + eo_woi['width'], eo_woi['x'], eo_woi['x']]
			ydata=[eo_woi['y'], eo_woi['y'], eo_woi['y'] + eo_woi['height'], eo_woi['y'] + eo_woi['height'], eo_woi['y']]
			plt.plot(xdata,ydata,color=WOI_COLOR)

			plt.colorbar(orientation='vertical') # location='bottom')
			'''
			# Ground truth confidence - to be replaced
			ax_conf=plt.subplot(322)
			ax_conf.set_title("Ground truth confidence")
			plt.imshow(data[...,qsf.GT_CONF], vmin=0, vmax=CONF_MAX, cmap='gray')
			if not lim_xy is None:
			pass # show frame
			xdata=[min(lim_xy[:2]),max(lim_xy[:2]),max(lim_xy[:2]),min(lim_xy[:2]),min(lim_xy[:2])]
			ydata=[min(lim_xy[2:]),min(lim_xy[2:]),max(lim_xy[2:]),max(lim_xy[2:]),min(lim_xy[2:])]
			plt.plot(xdata,ydata,color=WOI_COLOR)
			plt.colorbar(orientation='vertical') # location='bottom')
			'''

			ax_gtd=plt.subplot(321)
			ax_gtd.set_title("Ground truth disparity map")
			plt.imshow(data[...,qsf.GT_DISP], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
			setlimsxy(lim_xy)
			cross_out(plt, cross_out_mask)
			plt.colorbar(orientation='vertical') # location='bottom')

			ax_hed=plt.subplot(323)
			ax_hed.set_title("Heuristic disparity map")
			plt.imshow(data[...,qsf.HEUR_NAN], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
			setlimsxy(lim_xy)
			cross_out(plt, cross_out_mask)
			plt.colorbar(orientation='vertical') # location='bottom')

			ax_nnd=plt.subplot(325)
			ax_nnd.set_title("Network disparity output")
			plt.imshow(data[...,qsf.NN_NAN], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
			setlimsxy(lim_xy)
			cross_out(plt, cross_out_mask)
			plt.colorbar(orientation='vertical') # location='bottom')

			ax_hee=plt.subplot(324)
			ax_hee.set_title("Heuristic disparity error")
			cross_out(plt, cross_out_mask)
			plt.imshow(data[...,qsf.HEUR_DIFF], vmin=-ERR_AMPL, vmax=ERR_AMPL, cmap=cmap_diff)
			setlimsxy(lim_xy)
			cross_out(plt, cross_out_mask)
			plt.colorbar(orientation='vertical') # location='bottom')

			ax_nne=plt.subplot(326)
			ax_nne.set_title("Network disparity error")
			plt.imshow(data[...,qsf.NN_DIFF], vmin=-ERR_AMPL, vmax=ERR_AMPL, cmap=cmap_diff)
			setlimsxy(lim_xy)
			cross_out(plt, cross_out_mask)
			plt.colorbar(orientation='vertical') # location='bottom')

			plt.tight_layout(rect =[0,0,1,TIGHT_TOP], h_pad = TIGHT_HPAD, w_pad = TIGHT_WPAD)

			figs.append(fig)
			fb_noext = os.path.splitext(os.path.basename(img_file))[0]#
			# if subindex > 0:
			# if subindex < 10:
			# fb_noext+="abcdefghi"[subindex-1]
			# else:
			# fb_noext+="-"+str(subindex)
			ffiles.append(fb_noext)
			pass

			#
			#how to allow adjustment before applying tight_layout?
			pass
			for fig in figs:
			fig.tight_layout(rect =[0,0,1,TIGHT_TOP], h_pad = TIGHT_HPAD, w_pad = TIGHT_WPAD)

			if FIGS_EXTENSIONS and figs and 'save' in FIGS_SAVESHOW:
			try:
			print ("Creating output directory for figures: ",dirs['figures'])
			os.makedirs(dirs['figures'])
			except:
			pass
			pp=None
			if 'pdf' in FIGS_EXTENSIONS:
			if mode == 'infer':
			pdf_path = os.path.join(dirs['figures'],"figures-infer%s.pdf"%str(min_diff))
			else:
			pdf_path = os.path.join(dirs['figures'],"figures-train%s.pdf"%str(min_diff))
			pp= PdfPages(pdf_path)

			for fb_noext, fig in zip(ffiles,figs):
			for ext in FIGS_EXTENSIONS:
			if ext == 'pdf':
			pass
			fig.savefig(pp,format='pdf')
			else:
			if mode == 'infer':
			noext = fb_noext+'-infer'
			else:
			noext = fb_noext+'-train'

			fig.savefig(
			fname = os.path.join(dirs['figures'],noext+"."+ext),
			transparent = TRANSPARENT,
			)
			pass
			if pp:
			pp.close()


			if 'show' in FIGS_SAVESHOW:
			plt.show()

			#FIGS_ESXTENSIONS

			#qsf.evaluateAllResults(result_files = files['result'],
			# absolute_disparity = ABSOLUTE_DISPARITY,
			# cluster_radius = CLUSTER_RADIUS)
			print("All done")
			exit (0)

nn_eval_lwir.py

+36 −8

Original line number	Original line	Diff line number	Diff line
	@@ -18,6 +18,7 @@ import imagej_tiffwriter
	import time		import time
	import imagej_tiff as ijt		import imagej_tiff as ijt
	import matplotlib.pyplot as plt		import matplotlib.pyplot as plt
			import matplotlib.image as mpimg
	from matplotlib.backends.backend_pdf import PdfPages		from matplotlib.backends.backend_pdf import PdfPages
	import qcstereo_functions as qsf		import qcstereo_functions as qsf
	import numpy as np		import numpy as np
	@@ -156,11 +157,16 @@ def get_fig_params(disparity_ranges):
	extra_path = os.path.join(root_dir, dbg_parameters['extra'])		extra_path = os.path.join(root_dir, dbg_parameters['extra'])
	eo_width = dbg_parameters['eo_params']['width']		eo_width = dbg_parameters['eo_params']['width']
	eo_height = dbg_parameters['eo_params']['height']		eo_height = dbg_parameters['eo_params']['height']
	eo_woi = dbg_parameters['eo_params']['woi'] # (x,y,width, height)		eo_woi_rel = dbg_parameters['eo_params']['woi'] # (x,y,width, height)
			eo_woi={
			"x":eo_width * eo_woi_rel['x'],
			"y":eo_height * eo_woi_rel['y'],
			"width":eo_width * eo_woi_rel['width'],
			"height":eo_height * eo_woi_rel['height'],
			}
	eo_disparity_scale = 1.0/dbg_parameters['eo_params']['disparity_scale'] # 14.2		eo_disparity_scale = 1.0/dbg_parameters['eo_params']['disparity_scale'] # 14.2
	image_sets = dbg_parameters['extra_paths'] # list of dictionaries		image_sets = dbg_parameters['extra_paths'] # list of dictionaries


	pass		pass

	#temporary:		#temporary:
	@@ -251,6 +257,28 @@ for mode in modes: # ['train']:
	fig.canvas.set_window_title(img_pars['title'])		fig.canvas.set_window_title(img_pars['title'])
	fig.suptitle(img_pars['title'])		fig.suptitle(img_pars['title'])

			#insert LWIR JPEG image
			lwir_rgb = mpimg.imread(os.path.join(extra_path,img_pars['lwir_path']))
			ax_lwir = plt.subplot(421)
			ax_lwir.set_title("LWIR image (1 of 4)")
			plt.imshow(lwir_rgb)

			#insert EO JPEG image
			eo_rgb = mpimg.imread(os.path.join(extra_path,img_pars['eo_path']))
			ax_eo = plt.subplot(422)
			ax_eo.set_title("Visible range image (1 of 4)")
			plt.imshow(eo_rgb)
			if not eo_woi is None:
			eo_woi_jpeg={
			"x":eo_rgb.shape[1] * eo_woi_rel['x'],
			"y":eo_rgb.shape[0] * eo_woi_rel['y'],
			"width":eo_rgb.shape[1] * eo_woi_rel['width'],
			"height":eo_rgb.shape[0] * eo_woi_rel['height'],
			}
			xdata=[eo_woi_jpeg['x'], eo_woi_jpeg['x'] + eo_woi_jpeg['width'], eo_woi_jpeg['x'] + eo_woi_jpeg['width'], eo_woi_jpeg['x'], eo_woi_jpeg['x']]
			ydata=[eo_woi_jpeg['y'], eo_woi_jpeg['y'], eo_woi_jpeg['y'] + eo_woi_jpeg['height'], eo_woi_jpeg['y'] + eo_woi_jpeg['height'], eo_woi_jpeg['y']]
			plt.plot(xdata,ydata,color=WOI_COLOR)

	# Create EO DSI image		# Create EO DSI image
	# load tiff image		# load tiff image

	@@ -258,7 +286,7 @@ for mode in modes: # ['train']:
	ds_main = img_ds_main.image[...,img_pars['dsi_slice']] * eo_disparity_scale		ds_main = img_ds_main.image[...,img_pars['dsi_slice']] * eo_disparity_scale
	ds_main = np.maximum(ds_main, lim_val[0])		ds_main = np.maximum(ds_main, lim_val[0])
	ds_main = np.minimum(ds_main, lim_val[1])		ds_main = np.minimum(ds_main, lim_val[1])
	ax_conf=plt.subplot(322)		ax_conf=plt.subplot(424)
	ax_conf.set_title("Hi-res camera disparity map")		ax_conf.set_title("Hi-res camera disparity map")
	plt.imshow(ds_main, vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)		plt.imshow(ds_main, vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
	setlimsxy([-0.5, eo_width-0.5, -0.5, eo_height - 0.5])		setlimsxy([-0.5, eo_width-0.5, -0.5, eo_height - 0.5])
	@@ -282,28 +310,28 @@ for mode in modes: # ['train']:
	plt.colorbar(orientation='vertical') # location='bottom')		plt.colorbar(orientation='vertical') # location='bottom')
	'''		'''

	ax_gtd=plt.subplot(321)		ax_gtd=plt.subplot(423)
	ax_gtd.set_title("Ground truth disparity map")		ax_gtd.set_title("Ground truth disparity map")
	plt.imshow(data[...,qsf.GT_DISP], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)		plt.imshow(data[...,qsf.GT_DISP], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
	setlimsxy(lim_xy)		setlimsxy(lim_xy)
	cross_out(plt, cross_out_mask)		cross_out(plt, cross_out_mask)
	plt.colorbar(orientation='vertical') # location='bottom')		plt.colorbar(orientation='vertical') # location='bottom')

	ax_hed=plt.subplot(323)		ax_hed=plt.subplot(425)
	ax_hed.set_title("Heuristic disparity map")		ax_hed.set_title("Heuristic disparity map")
	plt.imshow(data[...,qsf.HEUR_NAN], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)		plt.imshow(data[...,qsf.HEUR_NAN], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
	setlimsxy(lim_xy)		setlimsxy(lim_xy)
	cross_out(plt, cross_out_mask)		cross_out(plt, cross_out_mask)
	plt.colorbar(orientation='vertical') # location='bottom')		plt.colorbar(orientation='vertical') # location='bottom')

	ax_nnd=plt.subplot(325)		ax_nnd=plt.subplot(427)
	ax_nnd.set_title("Network disparity output")		ax_nnd.set_title("Network disparity output")
	plt.imshow(data[...,qsf.NN_NAN], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)		plt.imshow(data[...,qsf.NN_NAN], vmin=lim_val[0], vmax=lim_val[1], cmap=cmap_disp)
	setlimsxy(lim_xy)		setlimsxy(lim_xy)
	cross_out(plt, cross_out_mask)		cross_out(plt, cross_out_mask)
	plt.colorbar(orientation='vertical') # location='bottom')		plt.colorbar(orientation='vertical') # location='bottom')

	ax_hee=plt.subplot(324)		ax_hee=plt.subplot(426)
	ax_hee.set_title("Heuristic disparity error")		ax_hee.set_title("Heuristic disparity error")
	cross_out(plt, cross_out_mask)		cross_out(plt, cross_out_mask)
	plt.imshow(data[...,qsf.HEUR_DIFF], vmin=-ERR_AMPL, vmax=ERR_AMPL, cmap=cmap_diff)		plt.imshow(data[...,qsf.HEUR_DIFF], vmin=-ERR_AMPL, vmax=ERR_AMPL, cmap=cmap_diff)
	@@ -311,7 +339,7 @@ for mode in modes: # ['train']:
	cross_out(plt, cross_out_mask)		cross_out(plt, cross_out_mask)
	plt.colorbar(orientation='vertical') # location='bottom')		plt.colorbar(orientation='vertical') # location='bottom')

	ax_nne=plt.subplot(326)		ax_nne=plt.subplot(428)
	ax_nne.set_title("Network disparity error")		ax_nne.set_title("Network disparity error")
	plt.imshow(data[...,qsf.NN_DIFF], vmin=-ERR_AMPL, vmax=ERR_AMPL, cmap=cmap_diff)		plt.imshow(data[...,qsf.NN_DIFF], vmin=-ERR_AMPL, vmax=ERR_AMPL, cmap=cmap_diff)
	setlimsxy(lim_xy)		setlimsxy(lim_xy)

Admin message