Skip to content

P
python3-imagej-tiff
Commit 464cacc9 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

Next versions

parent bb3cc9de
Show whitespace changes
Inline Side-by-side
Showing with 10285 additions and 31 deletions
explore_data.py
View file @ 464cacc9
...@@ -625,7 +625,16 @@ class ExploreData: ...@@ -625,7 +625,16 @@ class ExploreData:
if set_ds is None: if set_ds is None:
set_ds = self.train_ds set_ds = self.train_ds
try:
os.makedirs(os.path.dirname(tfr_filename))
print("Created directory "+os.path.dirname(tfr_filename))
except:
print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
pass
#skip writing if file exists - it will be possible to continue or run several instances
if os.path.exists(tfr_filename):
print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
return
writer = tf.python_io.TFRecordWriter(tfr_filename) writer = tf.python_io.TFRecordWriter(tfr_filename)
#$ files_list = [self.files_train, self.files_test][test_set] #$ files_list = [self.files_train, self.files_test][test_set]
seed_list = np.arange(len(files_list)) seed_list = np.arange(len(files_list))
...@@ -666,7 +675,7 @@ class ExploreData: ...@@ -666,7 +675,7 @@ class ExploreData:
example = tf.train.Example(features=tf.train.Features(feature=d_feature)) example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString()) writer.write(example.SerializeToString())
if (self.debug_level > 0): if (self.debug_level > 0):
print("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename)) print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))
writer.close() writer.close()
sys.stdout.flush() sys.stdout.flush()
...@@ -742,38 +751,30 @@ if __name__ == "__main__": ...@@ -742,38 +751,30 @@ if __name__ == "__main__":
try: try:
topdir_train = sys.argv[1] topdir_train = sys.argv[1]
except IndexError: except IndexError:
topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/" # topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
try: try:
topdir_test = sys.argv[2] topdir_test = sys.argv[2]
except IndexError: except IndexError:
topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/" # topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
topdir_test = "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
try: try:
pathTFR = sys.argv[3] pathTFR = sys.argv[3]
except IndexError: except IndexError:
pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/" # pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
try: try:
ml_subdir = sys.argv[4] ml_subdir = sys.argv[4]
except IndexError: except IndexError:
ml_subdir = "ml" # ml_subdir = "ml"
ml_subdir = "mlr32_18a"
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
scene = os.path.basename(test_corr)[:17]
scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
fname =scene+'-'+scene_version
img_filenameTFR = os.path.join(pathTFR,'img',fname)
writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
pass
exit(0)
#Parameters to generate neighbors data. Set radius to 0 to generate single-tile #Parameters to generate neighbors data. Set radius to 0 to generate single-tile
RADIUS = 1 RADIUS = 2 # 5x5
MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9 MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
VARIANCE_THRESHOLD = 1.5 VARIANCE_THRESHOLD = 1.5
NUM_TRAIN_SETS = 8 NUM_TRAIN_SETS = 8
...@@ -944,6 +945,13 @@ if __name__ == "__main__": ...@@ -944,6 +945,13 @@ if __name__ == "__main__":
fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD)) fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
plt.show() plt.show()
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
scene = os.path.basename(test_corr)[:17]
scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
fname =scene+'-'+scene_version
img_filenameTFR = os.path.join(pathTFR,'img',fname)
writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
pass pass
exit(0)
explore_data1.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
import os
import sys
import glob
import imagej_tiff as ijt
import numpy as np
import resource
import timeit
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter
import time
import tensorflow as tf
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
TIME_START = time.time()
TIME_LAST = TIME_START
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end)
TIME_LAST = t
def _dtype_feature(ndarray):
"""match appropriate tf.train.Feature class with dtype of ndarray. """
assert isinstance(ndarray, np.ndarray)
dtype_ = ndarray.dtype
if dtype_ == np.float64 or dtype_ == np.float32:
return lambda array: tf.train.Feature(float_list=tf.train.FloatList(value=array))
elif dtype_ == np.int64:
return lambda array: tf.train.Feature(int64_list=tf.train.Int64List(value=array))
else:
raise ValueError("The input should be numpy ndarray. \
Instead got {}".format(ndarray.dtype))
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecordDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append(np.array(example.features.feature['corr2d'] .float_list .value))
target_disparity_list.append(np.array(example.features.feature['target_disparity'] .float_list .value[0]))
gt_ds_list.append(np.array(example.features.feature['gt_ds'] .float_list .value))
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
return corr2d, target_disparity, gt_ds
def writeTFRewcordsImageTiles(img_path, tfr_filename): # test_set=False):
# train_filename = 'train.tfrecords' # address to save the TFRecords file
# open the TFRecords file
num_tiles = 242*324 # fixme
all_image_tiles = np.array(range(num_tiles))
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
img = ijt.imagej_tiff(test_corr, corr_layers, all_image_tiles)
corr2d = img.corr2d.reshape((num_tiles,-1))
target_disparity = img.target_disparity.reshape((num_tiles,-1))
gt_ds = img.gt_ds.reshape((num_tiles,-1))
if not '.tfrecords' in tfr_filename:
tfr_filename += '.tfrecords'
tfr_filename=tfr_filename.replace(' ','_')
try:
os.makedirs(os.path.dirname(tfr_filename))
except:
pass
writer = tf.python_io.TFRecordWriter(tfr_filename)
dtype_feature_corr2d = _dtype_feature(corr2d)
dtype_target_disparity = _dtype_feature(target_disparity)
dtype_feature_gt_ds = _dtype_feature(gt_ds)
for i in range(num_tiles):
x = corr2d[i].astype(np.float32)
y = target_disparity[i].astype(np.float32)
z = gt_ds[i].astype(np.float32)
d_feature = {'corr2d': dtype_feature_corr2d(x),
'target_disparity':dtype_target_disparity(y),
'gt_ds': dtype_feature_gt_ds(z)}
example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString())
pass
writer.close()
sys.stdout.flush()
class ExploreData:
PATTERN = "*-DSI_COMBO.tiff"
# ML_DIR = "ml"
ML_PATTERN = "*-ML_DATA*OFFS*.tiff"
"""
1527182801_296892-ML_DATARND-32B-O-FZ0.05-OFFS-0.20000_0.20000.tiff
"""
def getComboList(self, top_dir):
# patt = "*-DSI_COMBO.tiff"
tlist = []
for i in range(5):
pp = top_dir#) ,'**', patt) # works
for j in range (i):
pp = os.path.join(pp,'*')
pp = os.path.join(pp, ExploreData.PATTERN)
tlist += glob.glob(pp)
if (self.debug_level > 0):
print (pp+" "+str(len(tlist)))
if (self.debug_level > 0):
print("Found "+str(len(tlist))+" combo DSI files in "+top_dir+" :")
if (self.debug_level > 1):
print("\n".join(tlist))
return tlist
def loadComboFiles(self, tlist):
indx = 0
images = []
if (self.debug_level>2):
print(str(resource.getrusage(resource.RUSAGE_SELF)))
for combo_file in tlist:
tiff = ijt.imagej_tiff(combo_file,['disparity_rig','strength_rig'])
if not indx:
images = np.empty((len(tlist), tiff.image.shape[0],tiff.image.shape[1],tiff.image.shape[2]), tiff.image.dtype)
images[indx] = tiff.image
if (self.debug_level>2):
print(str(indx)+": "+str(resource.getrusage(resource.RUSAGE_SELF)))
indx += 1
return images
def getHistogramDSI(
self,
list_rds,
disparity_bins = 1000,
strength_bins = 100,
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 100.0,
disparity_max_clip = 100.0,
strength_min_drop = 0.1,
strength_min_clip = 0.1,
strength_max_drop = 1.0,
strength_max_clip = 0.9,
normalize = True,
no_histogram = False
):
good_tiles_list=[]
for combo_rds in list_rds:
good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
ds = combo_rds[ids]
disparity = ds[...,0]
strength = ds[...,1]
good_tiles[ids] = disparity >= disparity_min_drop
good_tiles[ids] &= disparity <= disparity_max_drop
good_tiles[ids] &= strength >= strength_min_drop
good_tiles[ids] &= strength <= strength_max_drop
disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
strength = np.nan_to_num(strength, copy = False) # likely should never happen
np.clip(disparity, disparity_min_clip, disparity_max_clip, out = disparity)
np.clip(strength, strength_min_clip, strength_max_clip, out = strength)
good_tiles_list.append(good_tiles)
combo_rds = np.concatenate(list_rds)
hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
x = combo_rds[...,1].flatten(),
y = combo_rds[...,0].flatten(),
bins= (strength_bins, disparity_bins),
range= ((strength_min_clip,strength_max_clip),(disparity_min_clip,disparity_max_clip)),
normed= normalize,
weights= np.concatenate(good_tiles_list).flatten())
for i, combo_rds in enumerate(list_rds):
for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
combo_rds[ids][...,1]*= good_tiles_list[i][ids]
return hist, xedges, yedges
def __init__(self,
topdir_train,
topdir_test,
ml_subdir,
debug_level = 0,
disparity_bins = 1000,
strength_bins = 100,
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 100.0,
disparity_max_clip = 100.0,
strength_min_drop = 0.1,
strength_min_clip = 0.1,
strength_max_drop = 1.0,
strength_max_clip = 0.9,
hist_sigma = 2.0, # Blur log histogram
hist_cutoff= 0.001 # of maximal
):
# file name
self.debug_level = debug_level
#self.testImageTiles()
self.disparity_bins = disparity_bins
self.strength_bins = strength_bins
self.disparity_min_drop = disparity_min_drop
self.disparity_min_clip = disparity_min_clip
self.disparity_max_drop = disparity_max_drop
self.disparity_max_clip = disparity_max_clip
self.strength_min_drop = strength_min_drop
self.strength_min_clip = strength_min_clip
self.strength_max_drop = strength_max_drop
self.strength_max_clip = strength_max_clip
self.hist_sigma = hist_sigma # Blur log histogram
self.hist_cutoff= hist_cutoff # of maximal
self.pre_log_offs = 0.001 # of histogram maximum
self.good_tiles = None
self.files_train = self.getComboList(topdir_train)
self.files_test = self.getComboList(topdir_test)
self.train_ds = self.loadComboFiles(self.files_train)
self.test_ds = self.loadComboFiles(self.files_test)
self.num_tiles = self.train_ds.shape[1]*self.train_ds.shape[2]
self.hist, xedges, yedges = self.getHistogramDSI(
list_rds = [self.train_ds,self.test_ds], # combo_rds,
disparity_bins = self.disparity_bins,
strength_bins = self.strength_bins,
disparity_min_drop = self.disparity_min_drop,
disparity_min_clip = self.disparity_min_clip,
disparity_max_drop = self.disparity_max_drop,
disparity_max_clip = self.disparity_max_clip,
strength_min_drop = self.strength_min_drop,
strength_min_clip = self.strength_min_clip,
strength_max_drop = self.strength_max_drop,
strength_max_clip = self.strength_max_clip,
normalize = True,
no_histogram = False
)
log_offset = self.pre_log_offs * self.hist.max()
h_cutoff = hist_cutoff * self.hist.max()
lhist = np.log(self.hist + log_offset)
blurred_lhist = gaussian_filter(lhist, sigma = self.hist_sigma)
self.blurred_hist = np.exp(blurred_lhist) - log_offset
self.good_tiles = self.blurred_hist >= h_cutoff
self.blurred_hist *= self.good_tiles # set bad ones to zero
def exploreNeibs(self,
data_ds, # disparity/strength data for all files (train or test)
radius, # how far to look from center each side ( 1- 3x3, 2 - 5x5)
disp_thesh = 5.0): # reduce effective variance for higher disparities
"""
For each tile calculate difference between max and min among neighbors and number of qualifying neighbors (bad cewnter is not removed)
"""
disp_min = np.empty_like(data_ds[...,0], dtype = np.float)
disp_max = np.empty_like(disp_min, dtype = np.float)
tile_neibs = np.zeros_like(disp_min, dtype = np.int)
dmin = data_ds[...,0].min()
dmax = data_ds[...,0].max()
good_tiles = self.getBB(data_ds) >= 0
side = 2 * radius + 1
for nf, ds in enumerate(data_ds):
disp = ds[...,0]
height = disp.shape[0]
width = disp.shape[1]
bad_max = np.ones((height+side, width+side), dtype=float) * dmax
bad_min = np.ones((height+side, width+side), dtype=float) * dmin
good = np.zeros((height+side, width+side), dtype=int)
#Assign centers of the array, replace bad tiles with max/min (so they will not change min/max)
bad_max[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmax)
bad_min[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmin)
good [radius:height+radius,radius:width+radius] = good_tiles[nf]
disp_min [nf,...] = disp
disp_max [nf,...] = disp
tile_neibs[nf,...] = good_tiles[nf]
for offset_y in range(-radius, radius+1):
oy = offset_y+radius
for offset_x in range(-radius, radius+1):
ox = offset_x+radius
if offset_y or offset_x: # Skip center - already copied
np.minimum(disp_min[nf], bad_max[oy:oy+height, ox:ox+width], out=disp_min[nf])
np.maximum(disp_max[nf], bad_min[oy:oy+height, ox:ox+width], out=disp_max[nf])
tile_neibs[nf] += good[oy:oy+height, ox:ox+width]
pass
pass
pass
pass
#disp_thesh
disp_avar = disp_max - disp_min
disp_rvar = disp_avar * disp_thesh / np.maximum(disp_max, 0.001) # removing division by 0 error - those tiles will be anyway discarded
disp_var = np.select([disp_max >= disp_thesh, disp_max < disp_thesh],[disp_rvar,disp_avar])
return disp_var, tile_neibs
def assignBatchBins(self,
disp_bins,
str_bins,
files_per_scene = 5, # not used here, will be used when generating batches
min_batch_choices=10, # not used here, will be used when generating batches
max_batch_files = 10): # not used here, will be used when generating batches
"""
for each disparity/strength combination (self.disparity_bins * self.strength_bins = 1000*100) provide number of "large"
variable-size disparity/strength bin, or -1 if this disparity/strength combination does not seem right
"""
self.files_per_scene = files_per_scene
self.min_batch_choices=min_batch_choices
self.max_batch_files = max_batch_files
hist_to_batch = np.zeros((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
hist_to_batch_multi = np.ones((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
scale_hist= (disp_bins * str_bins)/self.blurred_hist.sum()
norm_b_hist = self.blurred_hist * scale_hist
disp_list = [] # last disparity hist
# disp_multi = [] # number of disp rows to fit
disp_run_tot = 0.0
disp_batch = 0
disp=0
num_batch_bins = disp_bins * str_bins
disp_hist = np.linspace(0, num_batch_bins, disp_bins+1)
batch_index = 0
num_members = np.zeros((num_batch_bins,),int)
while disp_batch < disp_bins:
#disp_multi.append(1)
# while (disp < self.disparity_bins):
# disp_target_tot =disp_hist[disp_batch+1]
disp_run_tot_new = disp_run_tot
disp0 = disp # start disaprity matching disp_run_tot
while (disp_run_tot_new < disp_hist[disp_batch+1]) and (disp < self.disparity_bins):
disp_run_tot_new += norm_b_hist[:,disp].sum()
disp+=1;
disp_multi = 1
while (disp_batch < (disp_bins - 1)) and (disp_run_tot_new >= disp_hist[disp_batch+2]):
disp_batch += 1 # only if large disp_bins and very high hist value
disp_multi += 1
# now disp_run_tot - before this batch disparity col
str_bins_corr = str_bins * disp_multi # if too narrow disparity column - multiply number of strength columns
str_bins_corr_last = str_bins_corr -1
str_hist = np.linspace(disp_run_tot, disp_run_tot_new, str_bins_corr + 1)
str_run_tot_new = disp_run_tot
# str_batch = 0
str_index=0
# wide_col = norm_b_hist[:,disp0:disp] #disp0 - first column, disp - last+ 1
#iterate in linescan along the column
for si in range(self.strength_bins):
for di in range(disp0, disp,1):
if norm_b_hist[si,di] > 0.0 :
str_run_tot_new += norm_b_hist[si,di]
# do not increment after last to avoid precision issues
if (batch_index < num_batch_bins) and (num_members[batch_index] > 0) and (str_index < str_bins_corr_last) and (str_run_tot_new > str_hist[str_index+1]):
batch_index += 1
str_index += 1
if batch_index < num_batch_bins :
hist_to_batch[si,di] = batch_index
num_members[batch_index] += 1
else:
pass
else:
hist_to_batch[si,di] = -1
batch_index += 1 # it was not incremented afterthe last in the column to avoid rounding error
disp_batch += 1
disp_run_tot = disp_run_tot_new
pass
self.hist_to_batch = hist_to_batch
return hist_to_batch
def getBB(self, data_ds):
"""
for each file, each tile get histogram index (or -1 for bad tiles)
"""
hist_to_batch = self.hist_to_batch
files_batch_list = []
disp_step = ( self.disparity_max_clip - self.disparity_min_clip )/ self.disparity_bins
str_step = ( self.strength_max_clip - self.strength_min_clip )/ self.strength_bins
bb = np.empty_like(data_ds[...,0],dtype=int)
for findx in range(data_ds.shape[0]):
ds = data_ds[findx]
gt = ds[...,1] > 0.0 # OK
db = (((ds[...,0] - self.disparity_min_clip)/disp_step).astype(int))*gt
sb = (((ds[...,1] - self.strength_min_clip)/ str_step).astype(int))*gt
np.clip(db, 0, self.disparity_bins-1, out = db)
np.clip(sb, 0, self.strength_bins-1, out = sb)
bb[findx] = (self.hist_to_batch[sb.reshape(self.num_tiles),db.reshape(self.num_tiles)]) .reshape(db.shape[0],db.shape[1]) + (gt -1)
return bb
def makeBatchLists(self,
data_ds = None, # (disparity,strength) per scene, per tile
disp_var = None, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = None, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = None, # Minimal tile variance to include
max_var = None, # Maximal tile variance to include
min_neibs = None):# Minimal number of valid tiles to include
if data_ds is None:
data_ds = self.train_ds
hist_to_batch = self.hist_to_batch
num_batch_tiles = np.empty((data_ds.shape[0],self.hist_to_batch.max()+1),dtype = int)
bb = self.getBB(data_ds)
use_neibs = not ((disp_var is None) or (disp_neibs is None) or (min_var is None) or (max_var is None) or (min_neibs is None))
list_of_file_lists=[]
for findx in range(data_ds.shape[0]):
foffs = findx * self.num_tiles
lst = []
for i in range (self.hist_to_batch.max()+1):
lst.append([])
# bb1d = bb[findx].reshape(self.num_tiles)
if use_neibs:
disp_var_tiles = disp_var[findx].reshape(self.num_tiles)
disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
for n, indx in enumerate(bb[findx].reshape(self.num_tiles)):
if indx >= 0:
if use_neibs:
# disp_var_tiles = disp_var[findx].reshape(self.num_tiles)
# disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
if disp_neibs_tiles[n] < min_neibs:
continue # too few neighbors
if not disp_var_tiles[n] >= min_var:
continue #too small variance
if not disp_var_tiles[n] < max_var:
continue #too large variance
lst[indx].append(foffs + n)
lst_arr=[]
for i,l in enumerate(lst):
# lst_arr.append(np.array(l,dtype = int))
lst_arr.append(l)
num_batch_tiles[findx,i] = len(l)
list_of_file_lists.append(lst_arr)
self.list_of_file_lists= list_of_file_lists
self.num_batch_tiles = num_batch_tiles
return list_of_file_lists, num_batch_tiles
#todo: only use other files if there are no enough choices in the main file!
def augmentBatchFileIndices(self,
seed_index,
min_choices=None,
max_files = None,
set_ds = None
):
if min_choices is None:
min_choices = self.min_batch_choices
if max_files is None:
max_files = self.max_batch_files
if set_ds is None:
set_ds = self.train_ds
full_num_choices = self.num_batch_tiles[seed_index].copy()
flist = [seed_index]
all_choices = list(range(self.num_batch_tiles.shape[0]))
all_choices.remove(seed_index)
for _ in range (max_files-1):
if full_num_choices.min() >= min_choices:
break
findx = np.random.choice(all_choices)
flist.append(findx)
all_choices.remove(findx)
full_num_choices += self.num_batch_tiles[findx]
file_tiles_sparse = [[] for _ in set_ds] #list of empty lists for each train scene (will be sparse)
for nt in range(self.num_batch_tiles.shape[1]): #number of tiles per batch (not counting ml file variant)
tl = []
nchoices = 0
for findx in flist:
if (len(self.list_of_file_lists[findx][nt])):
tl.append(self.list_of_file_lists[findx][nt])
nchoices+= self.num_batch_tiles[findx][nt]
if nchoices >= min_choices: # use minimum of extra files
break;
tile = np.random.choice(np.concatenate(tl))
# print (nt, tile, tile//self.num_tiles, tile % self.num_tiles)
if not type (tile) is np.int64:
print("tile=",tile)
file_tiles_sparse[tile//self.num_tiles].append(tile % self.num_tiles)
file_tiles = []
for findx in flist:
file_tiles.append(np.sort(np.array(file_tiles_sparse[findx],dtype=int)))
return flist, file_tiles # file indices, list if tile indices for each file
def getMLList(self, ml_subdir, flist):
ml_list = []
for fn in flist:
ml_patt = os.path.join(os.path.dirname(fn), ml_subdir, ExploreData.ML_PATTERN)
ml_list.append(glob.glob(ml_patt))
## self.ml_list = ml_list
return ml_list
def getBatchData(
self,
flist,
tiles,
ml_list,
ml_num = None ): # 0 - use all ml files for the scene, >0 select random number
if ml_num is None:
ml_num = self.files_per_scene
ml_all_files = []
for findx in flist:
mli = list(range(len(ml_list[findx])))
if (ml_num > 0) and (ml_num < len(mli)):
mli_left = mli
mli = []
for _ in range(ml_num):
ml = np.random.choice(mli_left)
mli.append(ml)
mli_left.remove(ml)
ml_files = []
for ml_index in mli:
ml_files.append(ml_list[findx][ml_index])
ml_all_files.append(ml_files)
return ml_all_files
def prepareBatchData(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
if min_choices is None:
min_choices = self.min_batch_choices
if max_files is None:
max_files = self.max_batch_files
if ml_num is None:
ml_num = self.files_per_scene
if set_ds is None:
set_ds = self.train_ds
tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
height = set_ds.shape[1]
width = set_ds.shape[2]
width_m1 = width-1
height_m1 = height-1
# set_ds = [self.train_ds, self.test_ds][test_set]
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
# ml_all_files = self.getBatchData(flist, tiles, ml_list, ml_num) # 0 - use all ml files for the scene, >0 select random number
ml_all_files = self.getBatchData(flist, tiles, ml_list, 0) # ml_num) # 0 - use all ml files for the scene, >0 select random number
if self.debug_level > 1:
print ("==============",seed_index, flist)
for i, findx in enumerate(flist):
print(i,"\n".join(ml_all_files[i]))
print(tiles[i])
total_tiles = 0
for i, t in enumerate(tiles):
## total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
total_tiles += len(t) # tiles per scene * offset files per scene
if self.debug_level > 1:
print("Tiles in the batch=",total_tiles)
corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
gt_ds_batch = np.empty((total_tiles * tiles_in_sample, 2), dtype=float)
target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ), dtype=float)
start_tile = 0
for nscene, scene_files in enumerate(ml_all_files):
'''
Create tiles list including neighbors
'''
full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
indx = 0;
for i, nt in enumerate(tiles[nscene]):
ty = nt // width
tx = nt % width
for dy in range (-radius, radius+1):
y = np.clip(ty+dy,0,height_m1)
for dx in range (-radius, radius+1):
x = np.clip(tx+dx,0,width_m1)
full_tiles[indx] = y * width + x
indx += 1
"""
Assign tiles to several correlation files
"""
file_tiles = []
file_indices = []
for f in scene_files:
file_tiles.append([])
num_scene_files = len(scene_files)
for t in full_tiles:
fi = np.random.randint(0, num_scene_files)
file_tiles[fi].append(t)
file_indices.append(fi)
corr2d_list = []
target_disparity_list = []
gt_ds_list = []
for fi, path in enumerate (scene_files):
img = ijt.imagej_tiff(path, corr_layers, tile_list=file_tiles[fi])
corr2d_list.append (img.corr2d)
target_disparity_list.append(img.target_disparity)
gt_ds_list.append (img.gt_ds)
img_indices = [0] * len(scene_files)
for i, fi in enumerate(file_indices):
ti = img_indices[fi]
img_indices[fi] += 1
if corr2d_batch is None:
corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d_list[fi].shape[-1]))
gt_ds_batch [start_tile] = gt_ds_list[fi][ti]
target_disparity_batch [start_tile] = target_disparity_list[fi][ti]
corr2d_batch [start_tile] = corr2d_list[fi][ti]
start_tile += 1
"""
Sometimes get bad tile in ML file that was not bad in COMBO-DSI
Need to recover
np.argwhere(np.isnan(target_disparity_batch))
"""
bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
if (len(bad_tiles)>0):
print ("*** Got %d bad tiles in a batch, no code to replace :-("%(len(bad_tiles)))
# for now - just repeat some good tile
"""
for ibt in bad_tiles:
while np.isnan(target_disparity_batch[ibt]):
irt = np.random.randint(0,total_tiles)
if not np.isnan(target_disparity_batch[irt]):
target_disparity_batch[ibt] = target_disparity_batch[irt]
corr2d_batch[ibt] = corr2d_batch[irt]
gt_ds_batch[ibt] = gt_ds_batch[irt]
break
print (" done replacing")
"""
self.corr2d_batch = corr2d_batch
self.target_disparity_batch = target_disparity_batch
self.gt_ds_batch = gt_ds_batch
return corr2d_batch, target_disparity_batch, gt_ds_batch
def prepareBatchDataOld(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
if min_choices is None:
min_choices = self.min_batch_choices
if max_files is None:
max_files = self.max_batch_files
if ml_num is None:
ml_num = self.files_per_scene
if set_ds is None:
set_ds = self.train_ds
tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
height = set_ds.shape[1]
width = set_ds.shape[2]
width_m1 = width-1
height_m1 = height-1
# set_ds = [self.train_ds, self.test_ds][test_set]
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
ml_all_files = self.getBatchData(flist, tiles, ml_list, ml_num) # 0 - use all ml files for the scene, >0 select random number
if self.debug_level > 1:
print ("==============",seed_index, flist)
for i, findx in enumerate(flist):
print(i,"\n".join(ml_all_files[i]))
print(tiles[i])
total_tiles = 0
for i, t in enumerate(tiles):
total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
if self.debug_level > 1:
print("Tiles in the batch=",total_tiles)
corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
gt_ds_batch = np.empty((total_tiles * tiles_in_sample, 2), dtype=float)
target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ), dtype=float)
start_tile = 0
for nscene, scene_files in enumerate(ml_all_files):
for path in scene_files:
'''
Create tiles list including neighbors
'''
full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
indx = 0;
for i, nt in enumerate(tiles[nscene]):
ty = nt // width
tx = nt % width
for dy in range (-radius, radius+1):
y = np.clip(ty+dy,0,height_m1)
for dx in range (-radius, radius+1):
x = np.clip(tx+dx,0,width_m1)
full_tiles[indx] = y * width + x
indx += 1
#now tile_list is np.array instead of the list, but it seems to be OK
img = ijt.imagej_tiff(path, corr_layers, tile_list=full_tiles) # tiles[nscene])
corr2d = img.corr2d
target_disparity = img.target_disparity
gt_ds = img.gt_ds
end_tile = start_tile + corr2d.shape[0]
if corr2d_batch is None:
# corr2d_batch = np.empty((total_tiles, tiles_in_sample * len(corr_layers), corr2d.shape[-1]))
corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d.shape[-1]))
gt_ds_batch [start_tile:end_tile] = gt_ds
target_disparity_batch [start_tile:end_tile] = target_disparity
corr2d_batch [start_tile:end_tile] = corr2d
start_tile = end_tile
"""
Sometimes get bad tile in ML file that was not bad in COMBO-DSI
Need to recover
np.argwhere(np.isnan(target_disparity_batch))
"""
bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
if (len(bad_tiles)>0):
print ("*** Got %d bad tiles in a batch, replacing..."%(len(bad_tiles)), end=" ")
# for now - just repeat some good tile
for ibt in bad_tiles:
while np.isnan(target_disparity_batch[ibt]):
irt = np.random.randint(0,total_tiles)
if not np.isnan(target_disparity_batch[irt]):
target_disparity_batch[ibt] = target_disparity_batch[irt]
corr2d_batch[ibt] = corr2d_batch[irt]
gt_ds_batch[ibt] = gt_ds_batch[irt]
break
print (" done replacing")
self.corr2d_batch = corr2d_batch
self.target_disparity_batch = target_disparity_batch
self.gt_ds_batch = gt_ds_batch
return corr2d_batch, target_disparity_batch, gt_ds_batch
def writeTFRewcordsEpoch(self, tfr_filename, ml_list, files_list = None, set_ds= None, radius = 0): # test_set=False):
# train_filename = 'train.tfrecords' # address to save the TFRecords file
# open the TFRecords file
if not '.tfrecords' in tfr_filename:
tfr_filename += '.tfrecords'
tfr_filename=tfr_filename.replace(' ','_')
if files_list is None:
files_list = self.files_train
if set_ds is None:
set_ds = self.train_ds
try:
os.makedirs(os.path.dirname(tfr_filename))
print("Created directory "+os.path.dirname(tfr_filename))
except:
print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
pass
#skip writing if file exists - it will be possible to continue or run several instances
if os.path.exists(tfr_filename):
print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
return
writer = tf.python_io.TFRecordWriter(tfr_filename)
#$ files_list = [self.files_train, self.files_test][test_set]
seed_list = np.arange(len(files_list))
np.random.shuffle(seed_list)
cluster_size = (2 * radius + 1) * (2 * radius + 1)
for nscene, seed_index in enumerate(seed_list):
corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
#shuffles tiles in a batch
# tiles_in_batch = len(target_disparity_batch)
tiles_in_batch = corr2d_batch.shape[0]
clusters_in_batch = tiles_in_batch // cluster_size
# permut = np.random.permutation(tiles_in_batch)
permut = np.random.permutation(clusters_in_batch)
corr2d_clusters = corr2d_batch. reshape((clusters_in_batch,-1))
target_disparity_clusters = target_disparity_batch.reshape((clusters_in_batch,-1))
gt_ds_clusters = gt_ds_batch. reshape((clusters_in_batch,-1))
# corr2d_batch_shuffled = corr2d_batch[permut].reshape((corr2d_batch.shape[0], corr2d_batch.shape[1]*corr2d_batch.shape[2]))
# target_disparity_batch_shuffled = target_disparity_batch[permut].reshape((tiles_in_batch,1))
# gt_ds_batch_shuffled = gt_ds_batch[permut]
corr2d_batch_shuffled = corr2d_clusters[permut]. reshape((tiles_in_batch, -1))
target_disparity_batch_shuffled = target_disparity_clusters[permut].reshape((tiles_in_batch, -1))
gt_ds_batch_shuffled = gt_ds_clusters[permut]. reshape((tiles_in_batch, -1))
if nscene == 0:
dtype_feature_corr2d = _dtype_feature(corr2d_batch_shuffled)
dtype_target_disparity = _dtype_feature(target_disparity_batch_shuffled)
dtype_feature_gt_ds = _dtype_feature(gt_ds_batch_shuffled)
for i in range(tiles_in_batch):
x = corr2d_batch_shuffled[i].astype(np.float32)
y = target_disparity_batch_shuffled[i].astype(np.float32)
z = gt_ds_batch_shuffled[i].astype(np.float32)
d_feature = {'corr2d': dtype_feature_corr2d(x),
'target_disparity':dtype_target_disparity(y),
'gt_ds': dtype_feature_gt_ds(z)}
example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString())
if (self.debug_level > 0):
print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))
writer.close()
sys.stdout.flush()
def showVariance(self,
rds_list, # list of disparity/strength files, suchas training, testing
disp_var_list, # list of disparity variance files. Same shape(but last dim) as rds_list
num_neibs_list, # list of number of tile neibs files. Same shape(but last dim) as rds_list
variance_min = 0.0,
variance_max = 1.5,
neibs_min = 9,
#Same parameters as for the histogram
# disparity_bins = 1000,
# strength_bins = 100,
# disparity_min_drop = -0.1,
# disparity_min_clip = -0.1,
# disparity_max_drop = 100.0,
# disparity_max_clip = 100.0,
# strength_min_drop = 0.1,
# strength_min_clip = 0.1,
# strength_max_drop = 1.0,
# strength_max_clip = 0.9,
normalize = False): # True):
good_tiles_list=[]
for nf, combo_rds in enumerate(rds_list):
disp_var = disp_var_list[nf]
num_neibs = num_neibs_list[nf]
good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
ds = combo_rds[ids]
disparity = ds[...,0]
strength = ds[...,1]
variance = disp_var[ids]
neibs = num_neibs[ids]
good_tiles[ids] = disparity >= self.disparity_min_drop
good_tiles[ids] &= disparity <= self.disparity_max_drop
good_tiles[ids] &= strength >= self.strength_min_drop
good_tiles[ids] &= strength <= self.strength_max_drop
good_tiles[ids] &= neibs >= neibs_min
good_tiles[ids] &= variance >= variance_min
good_tiles[ids] &= variance < variance_max
disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
strength = np.nan_to_num(strength, copy = False) # likely should never happen
# np.clip(disparity, self.disparity_min_clip, self.disparity_max_clip, out = disparity)
# np.clip(strength, self.strength_min_clip, self.strength_max_clip, out = strength)
good_tiles_list.append(good_tiles)
combo_rds = np.concatenate(rds_list)
hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
x = combo_rds[...,1].flatten(),
y = combo_rds[...,0].flatten(),
bins= (self.strength_bins, self.disparity_bins),
range= ((self.strength_min_clip,self.strength_max_clip),(self.disparity_min_clip,self.disparity_max_clip)),
normed= normalize,
weights= np.concatenate(good_tiles_list).flatten())
mytitle = "Disparity_Strength variance histogram"
fig = plt.figure()
fig.canvas.set_window_title(mytitle)
fig.suptitle("Min variance = %f, max variance = %f, min neibs = %d"%(variance_min, variance_max, neibs_min))
# plt.imshow(hist, vmin=0, vmax=.1 * hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.imshow(hist, vmin=0.0, vmax=300.0)#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.colorbar(orientation='horizontal') # location='bottom')
# for i, combo_rds in enumerate(rds_list):
# for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
# combo_rds[ids][...,1]*= good_tiles_list[i][ids]
# return hist, xedges, yedges
#MAIN
if __name__ == "__main__":
try:
topdir_train = sys.argv[1]
except IndexError:
# topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
try:
topdir_test = sys.argv[2]
except IndexError:
# topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
topdir_test = "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
try:
pathTFR = sys.argv[3]
except IndexError:
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #no trailing "/"
try:
ml_subdir = sys.argv[4]
except IndexError:
# ml_subdir = "ml"
ml_subdir = "mlr32_18a"
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
#Parameters to generate neighbors data. Set radius to 0 to generate single-tile
RADIUS = 2 # 5x5
MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
VARIANCE_THRESHOLD = 1.5
NUM_TRAIN_SETS = 8
if RADIUS == 0:
BATCH_DISP_BINS = 50 # 1000 * 1
BATCH_STR_BINS = 20 # 10
elif RADIUS == 1:
BATCH_DISP_BINS = 15 # 120 * 9
BATCH_STR_BINS = 8
else: # RADIUS = 2
BATCH_DISP_BINS = 10 # 40 * 25
BATCH_STR_BINS = 4
train_filenameTFR = pathTFR+"/train"
test_filenameTFR = pathTFR+"/test"
# disp_bins = 20,
# str_bins=10)
# corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(train_filenameTFR)
# print_time("Read %d tiles"%(corr2d.shape[0]))
# exit (0)
ex_data = ExploreData(
topdir_train = topdir_train,
topdir_test = topdir_test,
ml_subdir = ml_subdir,
debug_level = 1, #3, ##0, #3,
disparity_bins = 200, #1000,
strength_bins = 100,
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 20.0, #100.0,
disparity_max_clip = 20.0, #100.0,
strength_min_drop = 0.1,
strength_min_clip = 0.1,
strength_max_drop = 1.0,
strength_max_clip = 0.9,
hist_sigma = 2.0, # Blur log histogram
hist_cutoff= 0.001) # of maximal
mytitle = "Disparity_Strength histogram"
fig = plt.figure()
fig.canvas.set_window_title(mytitle)
fig.suptitle(mytitle)
# plt.imshow(lhist,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.imshow(ex_data.blurred_hist, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.colorbar(orientation='horizontal') # location='bottom')
hist_to_batch = ex_data.assignBatchBins(
disp_bins = BATCH_DISP_BINS,
str_bins = BATCH_STR_BINS)
bb_display = hist_to_batch.copy()
bb_display = ( 1+ (bb_display % 2) + 2 * ((bb_display % 20)//10)) * (hist_to_batch > 0) #).astype(float)
fig2 = plt.figure()
fig2.canvas.set_window_title("Batch indices")
fig2.suptitle("Batch index for each disparity/strength cell")
plt.imshow(bb_display) #, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
""" prepare test dataset """
# RADIUS = 1
# MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid
# VARIANCE_THRESHOLD = 1.5
if (RADIUS > 0):
disp_var_test, num_neibs_test = ex_data.exploreNeibs(ex_data.test_ds, RADIUS)
disp_var_train, num_neibs_train = ex_data.exploreNeibs(ex_data.train_ds, RADIUS)
# show varinace histogram
# for var_thresh in [0.1, 1.0, 1.5, 2.0, 5.0]:
for var_thresh in [1.5]:
ex_data.showVariance(
rds_list = [ex_data.train_ds, ex_data.test_ds], # list of disparity/strength files, suchas training, testing
disp_var_list = [disp_var_train, disp_var_test], # list of disparity variance files. Same shape(but last dim) as rds_list
num_neibs_list = [num_neibs_train, num_neibs_test], # list of number of tile neibs files. Same shape(but last dim) as rds_list
variance_min = 0.0,
variance_max = var_thresh,
neibs_min = MIN_NEIBS)
ex_data.showVariance(
rds_list = [ex_data.train_ds, ex_data.test_ds], # list of disparity/strength files, suchas training, testing
disp_var_list = [disp_var_train, disp_var_test], # list of disparity variance files. Same shape(but last dim) as rds_list
num_neibs_list = [num_neibs_train, num_neibs_test], # list of number of tile neibs files. Same shape(but last dim) as rds_list
variance_min = var_thresh,
variance_max = 1000.0,
neibs_min = MIN_NEIBS)
pass
pass
else:
disp_var_test, num_neibs_test = None, None
disp_var_train, num_neibs_train = None, None
ml_list_train=ex_data.getMLList(ml_subdir, ex_data.files_train)
ml_list_test= ex_data.getMLList(ml_subdir, ex_data.files_test)
if RADIUS == 0 :
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.train_ds,
disp_var = disp_var_train, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_train, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
pass
# ex_data.makeBatchLists(data_ds = ex_data.train_ds)
for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
disp_var = disp_var_test, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_test, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
fpath = test_filenameTFR # +("-%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_test, set_ds= ex_data.test_ds)
pass
else: # RADIUS > 0
# train
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.train_ds,
disp_var = disp_var_train, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_train, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_le_train = num_batch_tiles_train.sum()
print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train))
for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("%03d_R%d_LE%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.train_ds,
disp_var = disp_var_train, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_train, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_gt_train = num_batch_tiles_train.sum()
high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train)
print("Number of > %f disparity variance tiles: %d, fraction = %f (train)"%(VARIANCE_THRESHOLD, num_gt_train, high_fract_train))
for train_var in range (NUM_TRAIN_SETS):
fpath = (train_filenameTFR+("%03d_R%d_GT%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
# test
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
disp_var = disp_var_test, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_test, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_le_test = num_batch_tiles_test.sum()
print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
fpath = test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
disp_var = disp_var_test, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_test, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_gt_test = num_batch_tiles_test.sum()
high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test))
fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
plt.show()
scene = os.path.basename(test_corr)[:17]
scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
fname =scene+'-'+scene_version
img_filenameTFR = os.path.join(pathTFR,'img',fname)
writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
pass
pass
explore_data2.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
import os
import sys
import glob
import imagej_tiff as ijt
import numpy as np
import resource
import timeit
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter
import time
import tensorflow as tf
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
TIME_START = time.time()
TIME_LAST = TIME_START
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end)
TIME_LAST = t
def _dtype_feature(ndarray):
"""match appropriate tf.train.Feature class with dtype of ndarray. """
assert isinstance(ndarray, np.ndarray)
dtype_ = ndarray.dtype
if dtype_ == np.float64 or dtype_ == np.float32:
return lambda array: tf.train.Feature(float_list=tf.train.FloatList(value=array))
elif dtype_ == np.int64:
return lambda array: tf.train.Feature(int64_list=tf.train.Int64List(value=array))
else:
raise ValueError("The input should be numpy ndarray. \
Instead got {}".format(ndarray.dtype))
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecordDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append(np.array(example.features.feature['corr2d'] .float_list .value))
target_disparity_list.append(np.array(example.features.feature['target_disparity'] .float_list .value[0]))
gt_ds_list.append(np.array(example.features.feature['gt_ds'] .float_list .value))
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
return corr2d, target_disparity, gt_ds
def writeTFRewcordsImageTiles(img_path, tfr_filename): # test_set=False):
# train_filename = 'train.tfrecords' # address to save the TFRecords file
# open the TFRecords file
num_tiles = 242*324 # fixme
all_image_tiles = np.array(range(num_tiles))
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
img = ijt.imagej_tiff(test_corr, corr_layers, all_image_tiles)
corr2d = img.corr2d.reshape((num_tiles,-1))
target_disparity = img.target_disparity.reshape((num_tiles,-1))
gt_ds = img.gt_ds.reshape((num_tiles,-1))
if not '.tfrecords' in tfr_filename:
tfr_filename += '.tfrecords'
tfr_filename=tfr_filename.replace(' ','_')
try:
os.makedirs(os.path.dirname(tfr_filename))
except:
pass
writer = tf.python_io.TFRecordWriter(tfr_filename)
dtype_feature_corr2d = _dtype_feature(corr2d)
dtype_target_disparity = _dtype_feature(target_disparity)
dtype_feature_gt_ds = _dtype_feature(gt_ds)
for i in range(num_tiles):
x = corr2d[i].astype(np.float32)
y = target_disparity[i].astype(np.float32)
z = gt_ds[i].astype(np.float32)
d_feature = {'corr2d': dtype_feature_corr2d(x),
'target_disparity':dtype_target_disparity(y),
'gt_ds': dtype_feature_gt_ds(z)}
example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString())
pass
writer.close()
sys.stdout.flush()
class ExploreData:
PATTERN = "*-DSI_COMBO.tiff"
# ML_DIR = "ml"
# ML_PATTERN = "*-ML_DATA*OFFS*.tiff"
ML_PATTERN = "*-ML_DATA*MAIN*.tiff"
# ML_PATTERN = "*-ML_DATA*OFFS-0.20000_0.20000.tiff"
"""
1527182801_296892-ML_DATARND-32B-O-FZ0.05-OFFS-0.20000_0.20000.tiff
"""
def getComboList(self, top_dir):
# patt = "*-DSI_COMBO.tiff"
tlist = []
for i in range(5):
pp = top_dir#) ,'**', patt) # works
for j in range (i):
pp = os.path.join(pp,'*')
pp = os.path.join(pp, ExploreData.PATTERN)
tlist += glob.glob(pp)
if (self.debug_level > 0):
print (pp+" "+str(len(tlist)))
if (self.debug_level > 0):
print("Found "+str(len(tlist))+" combo DSI files in "+top_dir+" :")
if (self.debug_level > 1):
print("\n".join(tlist))
return tlist
def loadComboFiles(self, tlist):
indx = 0
images = []
if (self.debug_level>2):
print(str(resource.getrusage(resource.RUSAGE_SELF)))
layers = ['disparity_rig','strength_rig','disparity_main']
for combo_file in tlist:
tiff = ijt.imagej_tiff(combo_file,layers)
if not indx:
images = np.empty((len(tlist), tiff.image.shape[0],tiff.image.shape[1],tiff.image.shape[2]), tiff.image.dtype)
images[indx] = tiff.image
if (self.debug_level>2):
print(str(indx)+": "+str(resource.getrusage(resource.RUSAGE_SELF)))
indx += 1
return images
def getHistogramDSI(
self,
list_rds,
disparity_bins = 1000,
strength_bins = 100,
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 100.0,
disparity_max_clip = 100.0,
strength_min_drop = 0.1,
strength_min_clip = 0.1,
strength_max_drop = 1.0,
strength_max_clip = 0.9,
max_main_offset = 0.0,
normalize = True,
no_histogram = False
):
good_tiles_list=[]
for combo_rds in list_rds:
good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
ds = combo_rds[ids]
disparity = ds[...,0]
strength = ds[...,1]
good_tiles[ids] = disparity >= disparity_min_drop
good_tiles[ids] &= disparity <= disparity_max_drop
good_tiles[ids] &= strength >= strength_min_drop
good_tiles[ids] &= strength <= strength_max_drop
if max_main_offset > 0.0:
disparity_main = ds[...,2]
good_tiles[ids] &= disparity_main <= (disparity + max_main_offset)
good_tiles[ids] &= disparity_main >= (disparity - max_main_offset)
disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
strength = np.nan_to_num(strength, copy = False) # likely should never happen
np.clip(disparity, disparity_min_clip, disparity_max_clip, out = disparity)
np.clip(strength, strength_min_clip, strength_max_clip, out = strength)
good_tiles_list.append(good_tiles)
combo_rds = np.concatenate(list_rds)
hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
x = combo_rds[...,1].flatten(),
y = combo_rds[...,0].flatten(),
bins= (strength_bins, disparity_bins),
range= ((strength_min_clip,strength_max_clip),(disparity_min_clip,disparity_max_clip)),
normed= normalize,
weights= np.concatenate(good_tiles_list).flatten())
for i, combo_rds in enumerate(list_rds):
for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
combo_rds[ids][...,1]*= good_tiles_list[i][ids]
return hist, xedges, yedges
def __init__(self,
topdir_train,
topdir_test,
ml_subdir,
max_main_offset = 2.0, # > 0.0 - do not use main camera tiles with offset more than this
debug_level = 0,
disparity_bins = 1000,
strength_bins = 100,
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 100.0,
disparity_max_clip = 100.0,
strength_min_drop = 0.1,
strength_min_clip = 0.1,
strength_max_drop = 1.0,
strength_max_clip = 0.9,
hist_sigma = 2.0, # Blur log histogram
hist_cutoff= 0.001 # of maximal
):
# file name
self.debug_level = debug_level
#self.testImageTiles()
self.max_main_offset = max_main_offset
self.disparity_bins = disparity_bins
self.strength_bins = strength_bins
self.disparity_min_drop = disparity_min_drop
self.disparity_min_clip = disparity_min_clip
self.disparity_max_drop = disparity_max_drop
self.disparity_max_clip = disparity_max_clip
self.strength_min_drop = strength_min_drop
self.strength_min_clip = strength_min_clip
self.strength_max_drop = strength_max_drop
self.strength_max_clip = strength_max_clip
self.hist_sigma = hist_sigma # Blur log histogram
self.hist_cutoff= hist_cutoff # of maximal
self.pre_log_offs = 0.001 # of histogram maximum
self.good_tiles = None
self.files_train = self.getComboList(topdir_train)
self.files_test = self.getComboList(topdir_test)
self.train_ds = self.loadComboFiles(self.files_train)
self.test_ds = self.loadComboFiles(self.files_test)
self.num_tiles = self.train_ds.shape[1]*self.train_ds.shape[2]
self.hist, xedges, yedges = self.getHistogramDSI(
list_rds = [self.train_ds,self.test_ds], # combo_rds,
disparity_bins = self.disparity_bins,
strength_bins = self.strength_bins,
disparity_min_drop = self.disparity_min_drop,
disparity_min_clip = self.disparity_min_clip,
disparity_max_drop = self.disparity_max_drop,
disparity_max_clip = self.disparity_max_clip,
strength_min_drop = self.strength_min_drop,
strength_min_clip = self.strength_min_clip,
strength_max_drop = self.strength_max_drop,
strength_max_clip = self.strength_max_clip,
max_main_offset = self.max_main_offset,
normalize = True,
no_histogram = False
)
log_offset = self.pre_log_offs * self.hist.max()
h_cutoff = hist_cutoff * self.hist.max()
lhist = np.log(self.hist + log_offset)
blurred_lhist = gaussian_filter(lhist, sigma = self.hist_sigma)
self.blurred_hist = np.exp(blurred_lhist) - log_offset
self.good_tiles = self.blurred_hist >= h_cutoff
self.blurred_hist *= self.good_tiles # set bad ones to zero
def exploreNeibs(self,
data_ds, # disparity/strength data for all files (train or test)
radius, # how far to look from center each side ( 1- 3x3, 2 - 5x5)
disp_thesh = 5.0): # reduce effective variance for higher disparities
"""
For each tile calculate difference between max and min among neighbors and number of qualifying neighbors (bad cewnter is not removed)
"""
disp_min = np.empty_like(data_ds[...,0], dtype = np.float)
disp_max = np.empty_like(disp_min, dtype = np.float)
tile_neibs = np.zeros_like(disp_min, dtype = np.int)
dmin = data_ds[...,0].min()
dmax = data_ds[...,0].max()
good_tiles = self.getBB(data_ds) >= 0
side = 2 * radius + 1
for nf, ds in enumerate(data_ds):
disp = ds[...,0]
height = disp.shape[0]
width = disp.shape[1]
bad_max = np.ones((height+side, width+side), dtype=float) * dmax
bad_min = np.ones((height+side, width+side), dtype=float) * dmin
good = np.zeros((height+side, width+side), dtype=int)
#Assign centers of the array, replace bad tiles with max/min (so they will not change min/max)
bad_max[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmax)
bad_min[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmin)
good [radius:height+radius,radius:width+radius] = good_tiles[nf]
disp_min [nf,...] = disp
disp_max [nf,...] = disp
tile_neibs[nf,...] = good_tiles[nf]
for offset_y in range(-radius, radius+1):
oy = offset_y+radius
for offset_x in range(-radius, radius+1):
ox = offset_x+radius
if offset_y or offset_x: # Skip center - already copied
np.minimum(disp_min[nf], bad_max[oy:oy+height, ox:ox+width], out=disp_min[nf])
np.maximum(disp_max[nf], bad_min[oy:oy+height, ox:ox+width], out=disp_max[nf])
tile_neibs[nf] += good[oy:oy+height, ox:ox+width]
pass
pass
pass
pass
#disp_thesh
disp_avar = disp_max - disp_min
disp_rvar = disp_avar * disp_thesh / np.maximum(disp_max, 0.001) # removing division by 0 error - those tiles will be anyway discarded
disp_var = np.select([disp_max >= disp_thesh, disp_max < disp_thesh],[disp_rvar,disp_avar])
return disp_var, tile_neibs
def assignBatchBins(self,
disp_bins,
str_bins,
files_per_scene = 5, # not used here, will be used when generating batches
min_batch_choices=10, # not used here, will be used when generating batches
max_batch_files = 10): # not used here, will be used when generating batches
"""
for each disparity/strength combination (self.disparity_bins * self.strength_bins = 1000*100) provide number of "large"
variable-size disparity/strength bin, or -1 if this disparity/strength combination does not seem right
"""
self.files_per_scene = files_per_scene
self.min_batch_choices=min_batch_choices
self.max_batch_files = max_batch_files
hist_to_batch = np.zeros((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
hist_to_batch_multi = np.ones((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
scale_hist= (disp_bins * str_bins)/self.blurred_hist.sum()
norm_b_hist = self.blurred_hist * scale_hist
disp_list = [] # last disparity hist
# disp_multi = [] # number of disp rows to fit
disp_run_tot = 0.0
disp_batch = 0
disp=0
num_batch_bins = disp_bins * str_bins
disp_hist = np.linspace(0, num_batch_bins, disp_bins+1)
batch_index = 0
num_members = np.zeros((num_batch_bins,),int)
while disp_batch < disp_bins:
#disp_multi.append(1)
# while (disp < self.disparity_bins):
# disp_target_tot =disp_hist[disp_batch+1]
disp_run_tot_new = disp_run_tot
disp0 = disp # start disaprity matching disp_run_tot
while (disp_run_tot_new < disp_hist[disp_batch+1]) and (disp < self.disparity_bins):
disp_run_tot_new += norm_b_hist[:,disp].sum()
disp+=1;
disp_multi = 1
while (disp_batch < (disp_bins - 1)) and (disp_run_tot_new >= disp_hist[disp_batch+2]):
disp_batch += 1 # only if large disp_bins and very high hist value
disp_multi += 1
# now disp_run_tot - before this batch disparity col
str_bins_corr = str_bins * disp_multi # if too narrow disparity column - multiply number of strength columns
str_bins_corr_last = str_bins_corr -1
str_hist = np.linspace(disp_run_tot, disp_run_tot_new, str_bins_corr + 1)
str_run_tot_new = disp_run_tot
# str_batch = 0
str_index=0
# wide_col = norm_b_hist[:,disp0:disp] #disp0 - first column, disp - last+ 1
#iterate in linescan along the column
for si in range(self.strength_bins):
for di in range(disp0, disp,1):
if norm_b_hist[si,di] > 0.0 :
str_run_tot_new += norm_b_hist[si,di]
# do not increment after last to avoid precision issues
if (batch_index < num_batch_bins) and (num_members[batch_index] > 0) and (str_index < str_bins_corr_last) and (str_run_tot_new > str_hist[str_index+1]):
batch_index += 1
str_index += 1
if batch_index < num_batch_bins :
hist_to_batch[si,di] = batch_index
num_members[batch_index] += 1
else:
pass
else:
hist_to_batch[si,di] = -1
batch_index += 1 # it was not incremented afterthe last in the column to avoid rounding error
disp_batch += 1
disp_run_tot = disp_run_tot_new
pass
self.hist_to_batch = hist_to_batch
return hist_to_batch
def getBB(self, data_ds):
"""
for each file, each tile get histogram index (or -1 for bad tiles)
"""
hist_to_batch = self.hist_to_batch
files_batch_list = []
disp_step = ( self.disparity_max_clip - self.disparity_min_clip )/ self.disparity_bins
str_step = ( self.strength_max_clip - self.strength_min_clip )/ self.strength_bins
bb = np.empty_like(data_ds[...,0],dtype=int)
for findx in range(data_ds.shape[0]):
ds = data_ds[findx]
gt = ds[...,1] > 0.0 # OK
db = (((ds[...,0] - self.disparity_min_clip)/disp_step).astype(int))*gt
sb = (((ds[...,1] - self.strength_min_clip)/ str_step).astype(int))*gt
np.clip(db, 0, self.disparity_bins-1, out = db)
np.clip(sb, 0, self.strength_bins-1, out = sb)
bb[findx] = (self.hist_to_batch[sb.reshape(self.num_tiles),db.reshape(self.num_tiles)]) .reshape(db.shape[0],db.shape[1]) + (gt -1)
return bb
def makeBatchLists(self,
data_ds = None, # (disparity,strength) per scene, per tile
disp_var = None, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = None, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = None, # Minimal tile variance to include
max_var = None, # Maximal tile variance to include
min_neibs = None):# Minimal number of valid tiles to include
if data_ds is None:
data_ds = self.train_ds
hist_to_batch = self.hist_to_batch
num_batch_tiles = np.empty((data_ds.shape[0],self.hist_to_batch.max()+1),dtype = int)
bb = self.getBB(data_ds)
use_neibs = not ((disp_var is None) or (disp_neibs is None) or (min_var is None) or (max_var is None) or (min_neibs is None))
list_of_file_lists=[]
for findx in range(data_ds.shape[0]):
foffs = findx * self.num_tiles
lst = []
for i in range (self.hist_to_batch.max()+1):
lst.append([])
# bb1d = bb[findx].reshape(self.num_tiles)
if use_neibs:
disp_var_tiles = disp_var[findx].reshape(self.num_tiles)
disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
for n, indx in enumerate(bb[findx].reshape(self.num_tiles)):
if indx >= 0:
if use_neibs:
# disp_var_tiles = disp_var[findx].reshape(self.num_tiles)
# disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
if disp_neibs_tiles[n] < min_neibs:
continue # too few neighbors
if not disp_var_tiles[n] >= min_var:
continue #too small variance
if not disp_var_tiles[n] < max_var:
continue #too large variance
lst[indx].append(foffs + n)
lst_arr=[]
for i,l in enumerate(lst):
# lst_arr.append(np.array(l,dtype = int))
lst_arr.append(l)
num_batch_tiles[findx,i] = len(l)
list_of_file_lists.append(lst_arr)
self.list_of_file_lists= list_of_file_lists
self.num_batch_tiles = num_batch_tiles
return list_of_file_lists, num_batch_tiles
#todo: only use other files if there are no enough choices in the main file!
def augmentBatchFileIndices(self,
seed_index,
min_choices=None,
max_files = None,
set_ds = None
):
if min_choices is None:
min_choices = self.min_batch_choices
if max_files is None:
max_files = self.max_batch_files
if set_ds is None:
set_ds = self.train_ds
full_num_choices = self.num_batch_tiles[seed_index].copy()
flist = [seed_index]
all_choices = list(range(self.num_batch_tiles.shape[0]))
all_choices.remove(seed_index)
for _ in range (max_files-1):
if full_num_choices.min() >= min_choices:
break
findx = np.random.choice(all_choices)
flist.append(findx)
all_choices.remove(findx)
full_num_choices += self.num_batch_tiles[findx]
file_tiles_sparse = [[] for _ in set_ds] #list of empty lists for each train scene (will be sparse)
for nt in range(self.num_batch_tiles.shape[1]): #number of tiles per batch (not counting ml file variant) // radius2 - 40
tl = []
nchoices = 0
for findx in flist:
if (len(self.list_of_file_lists[findx][nt])):
tl.append(self.list_of_file_lists[findx][nt])
nchoices+= self.num_batch_tiles[findx][nt]
if nchoices >= min_choices: # use minimum of extra files
break;
while len(tl)==0:
print("** BUG! could not find a single candidate from files ",flist," for cell "+nt)
print("trying to use some other cell")
nt1 = np.random.randint(0,self.num_batch_tiles.shape[1])
for findx in flist:
if (len(self.list_of_file_lists[findx][nt1])):
tl.append(self.list_of_file_lists[findx][nt1])
nchoices+= self.num_batch_tiles[findx][nt1]
if nchoices >= min_choices: # use minimum of extra files
break;
tile = np.random.choice(np.concatenate(tl))
"""
Traceback (most recent call last):
File "explore_data2.py", line 1041, in <module>
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
File "explore_data2.py", line 761, in writeTFRewcordsEpoch
corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
File "explore_data2.py", line 556, in prepareBatchData
flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
File "explore_data2.py", line 494, in augmentBatchFileIndices
tile = np.random.choice(np.concatenate(tl))
ValueError: need at least one array to concatenate
"""
# print (nt, tile, tile//self.num_tiles, tile % self.num_tiles)
if not type (tile) is np.int64:
print("tile=",tile)
file_tiles_sparse[tile//self.num_tiles].append(tile % self.num_tiles)
file_tiles = []
for findx in flist:
file_tiles.append(np.sort(np.array(file_tiles_sparse[findx],dtype=int)))
return flist, file_tiles # file indices, list if tile indices for each file
def getMLList(self, ml_subdir, flist):
ml_list = []
for fn in flist:
ml_patt = os.path.join(os.path.dirname(fn), ml_subdir, ExploreData.ML_PATTERN)
ml_list.append(glob.glob(ml_patt))
## self.ml_list = ml_list
return ml_list
def getBatchData(
self,
flist,
tiles,
ml_list,
ml_num = None ): # 0 - use all ml files for the scene, >0 select random number
if ml_num is None:
ml_num = self.files_per_scene
ml_all_files = []
for findx in flist:
mli = list(range(len(ml_list[findx])))
if (ml_num > 0) and (ml_num < len(mli)):
mli_left = mli
mli = []
for _ in range(ml_num):
ml = np.random.choice(mli_left)
mli.append(ml)
mli_left.remove(ml)
ml_files = []
for ml_index in mli:
ml_files.append(ml_list[findx][ml_index])
ml_all_files.append(ml_files)
return ml_all_files
def prepareBatchData(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
if min_choices is None:
min_choices = self.min_batch_choices
if max_files is None:
max_files = self.max_batch_files
if ml_num is None:
ml_num = self.files_per_scene
if set_ds is None:
set_ds = self.train_ds
tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
height = set_ds.shape[1]
width = set_ds.shape[2]
width_m1 = width-1
height_m1 = height-1
# set_ds = [self.train_ds, self.test_ds][test_set]
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
# ml_all_files = self.getBatchData(flist, tiles, ml_list, ml_num) # 0 - use all ml files for the scene, >0 select random number
ml_all_files = self.getBatchData(flist, tiles, ml_list, 0) # ml_num) # 0 - use all ml files for the scene, >0 select random number
if self.debug_level > 1:
print ("==============",seed_index, flist)
for i, findx in enumerate(flist):
print(i,"\n".join(ml_all_files[i]))
print(tiles[i])
total_tiles = 0
for i, t in enumerate(tiles):
## total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
total_tiles += len(t) # tiles per scene * offset files per scene
if self.debug_level > 1:
print("Tiles in the batch=",total_tiles)
corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
gt_ds_batch = np.empty((total_tiles * tiles_in_sample, 2), dtype=float)
target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ), dtype=float)
start_tile = 0
for nscene, scene_files in enumerate(ml_all_files):
'''
Create tiles list including neighbors
'''
full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
indx = 0;
for i, nt in enumerate(tiles[nscene]):
ty = nt // width
tx = nt % width
for dy in range (-radius, radius+1):
y = np.clip(ty+dy,0,height_m1)
for dx in range (-radius, radius+1):
x = np.clip(tx+dx,0,width_m1)
full_tiles[indx] = y * width + x
indx += 1
"""
Assign tiles to several correlation files
"""
file_tiles = []
file_indices = []
for f in scene_files:
file_tiles.append([])
num_scene_files = len(scene_files)
for t in full_tiles:
fi = np.random.randint(0, num_scene_files)
file_tiles[fi].append(t)
file_indices.append(fi)
corr2d_list = []
target_disparity_list = []
gt_ds_list = []
for fi, path in enumerate (scene_files):
img = ijt.imagej_tiff(path, corr_layers, tile_list=file_tiles[fi])
corr2d_list.append (img.corr2d)
target_disparity_list.append(img.target_disparity)
gt_ds_list.append (img.gt_ds)
img_indices = [0] * len(scene_files)
for i, fi in enumerate(file_indices):
ti = img_indices[fi]
img_indices[fi] += 1
if corr2d_batch is None:
corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d_list[fi].shape[-1]))
gt_ds_batch [start_tile] = gt_ds_list[fi][ti]
target_disparity_batch [start_tile] = target_disparity_list[fi][ti]
corr2d_batch [start_tile] = corr2d_list[fi][ti]
start_tile += 1
"""
Sometimes get bad tile in ML file that was not bad in COMBO-DSI
Need to recover
np.argwhere(np.isnan(target_disparity_batch))
"""
bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
if (len(bad_tiles)>0):
print ("*** Got %d bad tiles in a batch, no code to replace :-("%(len(bad_tiles)))
# for now - just repeat some good tile
"""
for ibt in bad_tiles:
while np.isnan(target_disparity_batch[ibt]):
irt = np.random.randint(0,total_tiles)
if not np.isnan(target_disparity_batch[irt]):
target_disparity_batch[ibt] = target_disparity_batch[irt]
corr2d_batch[ibt] = corr2d_batch[irt]
gt_ds_batch[ibt] = gt_ds_batch[irt]
break
print (" done replacing")
"""
self.corr2d_batch = corr2d_batch
self.target_disparity_batch = target_disparity_batch
self.gt_ds_batch = gt_ds_batch
return corr2d_batch, target_disparity_batch, gt_ds_batch
def prepareBatchDataOld(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
if min_choices is None:
min_choices = self.min_batch_choices
if max_files is None:
max_files = self.max_batch_files
if ml_num is None:
ml_num = self.files_per_scene
if set_ds is None:
set_ds = self.train_ds
tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
height = set_ds.shape[1]
width = set_ds.shape[2]
width_m1 = width-1
height_m1 = height-1
# set_ds = [self.train_ds, self.test_ds][test_set]
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
ml_all_files = self.getBatchData(flist, tiles, ml_list, ml_num) # 0 - use all ml files for the scene, >0 select random number
if self.debug_level > 1:
print ("==============",seed_index, flist)
for i, findx in enumerate(flist):
print(i,"\n".join(ml_all_files[i]))
print(tiles[i])
total_tiles = 0
for i, t in enumerate(tiles):
total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
if self.debug_level > 1:
print("Tiles in the batch=",total_tiles)
corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
gt_ds_batch = np.empty((total_tiles * tiles_in_sample, 2), dtype=float)
target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ), dtype=float)
start_tile = 0
for nscene, scene_files in enumerate(ml_all_files):
for path in scene_files:
'''
Create tiles list including neighbors
'''
full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
indx = 0;
for i, nt in enumerate(tiles[nscene]):
ty = nt // width
tx = nt % width
for dy in range (-radius, radius+1):
y = np.clip(ty+dy,0,height_m1)
for dx in range (-radius, radius+1):
x = np.clip(tx+dx,0,width_m1)
full_tiles[indx] = y * width + x
indx += 1
#now tile_list is np.array instead of the list, but it seems to be OK
img = ijt.imagej_tiff(path, corr_layers, tile_list=full_tiles) # tiles[nscene])
corr2d = img.corr2d
target_disparity = img.target_disparity
gt_ds = img.gt_ds
end_tile = start_tile + corr2d.shape[0]
if corr2d_batch is None:
# corr2d_batch = np.empty((total_tiles, tiles_in_sample * len(corr_layers), corr2d.shape[-1]))
corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d.shape[-1]))
gt_ds_batch [start_tile:end_tile] = gt_ds
target_disparity_batch [start_tile:end_tile] = target_disparity
corr2d_batch [start_tile:end_tile] = corr2d
start_tile = end_tile
"""
Sometimes get bad tile in ML file that was not bad in COMBO-DSI
Need to recover
np.argwhere(np.isnan(target_disparity_batch))
"""
bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
if (len(bad_tiles)>0):
print ("*** Got %d bad tiles in a batch, replacing..."%(len(bad_tiles)), end=" ")
# for now - just repeat some good tile
for ibt in bad_tiles:
while np.isnan(target_disparity_batch[ibt]):
irt = np.random.randint(0,total_tiles)
if not np.isnan(target_disparity_batch[irt]):
target_disparity_batch[ibt] = target_disparity_batch[irt]
corr2d_batch[ibt] = corr2d_batch[irt]
gt_ds_batch[ibt] = gt_ds_batch[irt]
break
print (" done replacing")
self.corr2d_batch = corr2d_batch
self.target_disparity_batch = target_disparity_batch
self.gt_ds_batch = gt_ds_batch
return corr2d_batch, target_disparity_batch, gt_ds_batch
def writeTFRewcordsEpoch(self, tfr_filename, ml_list, files_list = None, set_ds= None, radius = 0): # test_set=False):
# train_filename = 'train.tfrecords' # address to save the TFRecords file
# open the TFRecords file
if not '.tfrecords' in tfr_filename:
tfr_filename += '.tfrecords'
tfr_filename=tfr_filename.replace(' ','_')
if files_list is None:
files_list = self.files_train
if set_ds is None:
set_ds = self.train_ds
try:
os.makedirs(os.path.dirname(tfr_filename))
print("Created directory "+os.path.dirname(tfr_filename))
except:
print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
pass
#skip writing if file exists - it will be possible to continue or run several instances
if os.path.exists(tfr_filename):
print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
return
writer = tf.python_io.TFRecordWriter(tfr_filename)
#$ files_list = [self.files_train, self.files_test][test_set]
seed_list = np.arange(len(files_list))
np.random.shuffle(seed_list)
cluster_size = (2 * radius + 1) * (2 * radius + 1)
for nscene, seed_index in enumerate(seed_list):
corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
#shuffles tiles in a batch
# tiles_in_batch = len(target_disparity_batch)
tiles_in_batch = corr2d_batch.shape[0]
clusters_in_batch = tiles_in_batch // cluster_size
# permut = np.random.permutation(tiles_in_batch)
permut = np.random.permutation(clusters_in_batch)
corr2d_clusters = corr2d_batch. reshape((clusters_in_batch,-1))
target_disparity_clusters = target_disparity_batch.reshape((clusters_in_batch,-1))
gt_ds_clusters = gt_ds_batch. reshape((clusters_in_batch,-1))
# corr2d_batch_shuffled = corr2d_batch[permut].reshape((corr2d_batch.shape[0], corr2d_batch.shape[1]*corr2d_batch.shape[2]))
# target_disparity_batch_shuffled = target_disparity_batch[permut].reshape((tiles_in_batch,1))
# gt_ds_batch_shuffled = gt_ds_batch[permut]
corr2d_batch_shuffled = corr2d_clusters[permut]. reshape((tiles_in_batch, -1))
target_disparity_batch_shuffled = target_disparity_clusters[permut].reshape((tiles_in_batch, -1))
gt_ds_batch_shuffled = gt_ds_clusters[permut]. reshape((tiles_in_batch, -1))
if nscene == 0:
dtype_feature_corr2d = _dtype_feature(corr2d_batch_shuffled)
dtype_target_disparity = _dtype_feature(target_disparity_batch_shuffled)
dtype_feature_gt_ds = _dtype_feature(gt_ds_batch_shuffled)
for i in range(tiles_in_batch):
x = corr2d_batch_shuffled[i].astype(np.float32)
y = target_disparity_batch_shuffled[i].astype(np.float32)
z = gt_ds_batch_shuffled[i].astype(np.float32)
d_feature = {'corr2d': dtype_feature_corr2d(x),
'target_disparity':dtype_target_disparity(y),
'gt_ds': dtype_feature_gt_ds(z)}
example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString())
if (self.debug_level > 0):
print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))
writer.close()
sys.stdout.flush()
def showVariance(self,
rds_list, # list of disparity/strength files, suchas training, testing
disp_var_list, # list of disparity variance files. Same shape(but last dim) as rds_list
num_neibs_list, # list of number of tile neibs files. Same shape(but last dim) as rds_list
variance_min = 0.0,
variance_max = 1.5,
neibs_min = 9,
#Same parameters as for the histogram
# disparity_bins = 1000,
# strength_bins = 100,
# disparity_min_drop = -0.1,
# disparity_min_clip = -0.1,
# disparity_max_drop = 100.0,
# disparity_max_clip = 100.0,
# strength_min_drop = 0.1,
# strength_min_clip = 0.1,
# strength_max_drop = 1.0,
# strength_max_clip = 0.9,
normalize = False): # True):
good_tiles_list=[]
for nf, combo_rds in enumerate(rds_list):
disp_var = disp_var_list[nf]
num_neibs = num_neibs_list[nf]
good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
ds = combo_rds[ids]
disparity = ds[...,0]
strength = ds[...,1]
variance = disp_var[ids]
neibs = num_neibs[ids]
good_tiles[ids] = disparity >= self.disparity_min_drop
good_tiles[ids] &= disparity <= self.disparity_max_drop
good_tiles[ids] &= strength >= self.strength_min_drop
good_tiles[ids] &= strength <= self.strength_max_drop
good_tiles[ids] &= neibs >= neibs_min
good_tiles[ids] &= variance >= variance_min
good_tiles[ids] &= variance < variance_max
disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
strength = np.nan_to_num(strength, copy = False) # likely should never happen
# np.clip(disparity, self.disparity_min_clip, self.disparity_max_clip, out = disparity)
# np.clip(strength, self.strength_min_clip, self.strength_max_clip, out = strength)
good_tiles_list.append(good_tiles)
combo_rds = np.concatenate(rds_list)
hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
x = combo_rds[...,1].flatten(),
y = combo_rds[...,0].flatten(),
bins= (self.strength_bins, self.disparity_bins),
range= ((self.strength_min_clip,self.strength_max_clip),(self.disparity_min_clip,self.disparity_max_clip)),
normed= normalize,
weights= np.concatenate(good_tiles_list).flatten())
mytitle = "Disparity_Strength variance histogram"
fig = plt.figure()
fig.canvas.set_window_title(mytitle)
fig.suptitle("Min variance = %f, max variance = %f, min neibs = %d"%(variance_min, variance_max, neibs_min))
# plt.imshow(hist, vmin=0, vmax=.1 * hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.imshow(hist, vmin=0.0, vmax=300.0)#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.colorbar(orientation='horizontal') # location='bottom')
# for i, combo_rds in enumerate(rds_list):
# for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
# combo_rds[ids][...,1]*= good_tiles_list[i][ids]
# return hist, xedges, yedges
#MAIN
if __name__ == "__main__":
try:
topdir_train = sys.argv[1]
except IndexError:
# topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
try:
topdir_test = sys.argv[2]
except IndexError:
# topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
topdir_test = "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
try:
pathTFR = sys.argv[3]
except IndexError:
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_4" #no trailing "/"
try:
ml_subdir = sys.argv[4]
except IndexError:
# ml_subdir = "ml"
ml_subdir = "mlr32_18a"
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
test_corr = '/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527256816_150165/v02/mlr32_18a/1527256816_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
# test_corr = '/home/eyesis/x3d_data/models/dsi_combo_and_ml_all/state_street/1527256858_150165/v01/mlr32_18a/1527256858_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
#Parameters to generate neighbors data. Set radius to 0 to generate single-tile
RADIUS = 2 # 5x5
MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
VARIANCE_THRESHOLD = 1.5
NUM_TRAIN_SETS = 24 # 8
if RADIUS == 0:
BATCH_DISP_BINS = 50 # 1000 * 1
BATCH_STR_BINS = 20 # 10
elif RADIUS == 1:
BATCH_DISP_BINS = 15 # 120 * 9
BATCH_STR_BINS = 8
else: # RADIUS = 2
BATCH_DISP_BINS = 10 # 40 * 25
BATCH_STR_BINS = 4
train_filenameTFR = pathTFR+"/train"
test_filenameTFR = pathTFR+"/test"
# disp_bins = 20,
# str_bins=10)
# corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(train_filenameTFR)
# print_time("Read %d tiles"%(corr2d.shape[0]))
# exit (0)
ex_data = ExploreData(
topdir_train = topdir_train,
topdir_test = topdir_test,
ml_subdir = ml_subdir,
debug_level = 1, #3, ##0, #3,
disparity_bins = 200, #1000,
strength_bins = 100,
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 20.0, #100.0,
disparity_max_clip = 20.0, #100.0,
strength_min_drop = 0.1,
strength_min_clip = 0.1,
strength_max_drop = 1.0,
strength_max_clip = 0.9,
hist_sigma = 2.0, # Blur log histogram
hist_cutoff= 0.001) # of maximal
mytitle = "Disparity_Strength histogram"
fig = plt.figure()
fig.canvas.set_window_title(mytitle)
fig.suptitle(mytitle)
# plt.imshow(lhist,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.imshow(ex_data.blurred_hist, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
plt.colorbar(orientation='horizontal') # location='bottom')
hist_to_batch = ex_data.assignBatchBins(
disp_bins = BATCH_DISP_BINS,
str_bins = BATCH_STR_BINS)
bb_display = hist_to_batch.copy()
bb_display = ( 1+ (bb_display % 2) + 2 * ((bb_display % 20)//10)) * (hist_to_batch > 0) #).astype(float)
fig2 = plt.figure()
fig2.canvas.set_window_title("Batch indices")
fig2.suptitle("Batch index for each disparity/strength cell")
plt.imshow(bb_display) #, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
""" prepare test dataset """
# RADIUS = 1
# MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid
# VARIANCE_THRESHOLD = 1.5
if (RADIUS > 0):
disp_var_test, num_neibs_test = ex_data.exploreNeibs(ex_data.test_ds, RADIUS)
disp_var_train, num_neibs_train = ex_data.exploreNeibs(ex_data.train_ds, RADIUS)
# show varinace histogram
# for var_thresh in [0.1, 1.0, 1.5, 2.0, 5.0]:
for var_thresh in [1.5]:
ex_data.showVariance(
rds_list = [ex_data.train_ds, ex_data.test_ds], # list of disparity/strength files, suchas training, testing
disp_var_list = [disp_var_train, disp_var_test], # list of disparity variance files. Same shape(but last dim) as rds_list
num_neibs_list = [num_neibs_train, num_neibs_test], # list of number of tile neibs files. Same shape(but last dim) as rds_list
variance_min = 0.0,
variance_max = var_thresh,
neibs_min = MIN_NEIBS)
ex_data.showVariance(
rds_list = [ex_data.train_ds, ex_data.test_ds], # list of disparity/strength files, suchas training, testing
disp_var_list = [disp_var_train, disp_var_test], # list of disparity variance files. Same shape(but last dim) as rds_list
num_neibs_list = [num_neibs_train, num_neibs_test], # list of number of tile neibs files. Same shape(but last dim) as rds_list
variance_min = var_thresh,
variance_max = 1000.0,
neibs_min = MIN_NEIBS)
pass
pass
else:
disp_var_test, num_neibs_test = None, None
disp_var_train, num_neibs_train = None, None
ml_list_train=ex_data.getMLList(ml_subdir, ex_data.files_train)
ml_list_test= ex_data.getMLList(ml_subdir, ex_data.files_test)
if RADIUS == 0 :
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.train_ds,
disp_var = disp_var_train, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_train, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
pass
# ex_data.makeBatchLists(data_ds = ex_data.train_ds)
for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
disp_var = disp_var_test, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_test, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
fpath = test_filenameTFR # +("-%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_test, set_ds= ex_data.test_ds)
pass
else: # RADIUS > 0
# train
for train_var in range (NUM_TRAIN_SETS): # Recalculate list for each file - slower, but will alternate lvar/hvar
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.train_ds,
disp_var = disp_var_train, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_train, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_le_train = num_batch_tiles_train.sum()
print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train))
# for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("%03d_R%d_LE%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.train_ds,
disp_var = disp_var_train, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_train, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_gt_train = num_batch_tiles_train.sum()
high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train)
print("Number of > %f disparity variance tiles: %d, fraction = %f (train)"%(VARIANCE_THRESHOLD, num_gt_train, high_fract_train))
# for train_var in range (NUM_TRAIN_SETS):
fpath = (train_filenameTFR+("%03d_R%d_GT%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
if train_var < 1: # make test files immediately after the train ones
# test
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
disp_var = disp_var_test, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_test, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_le_test = num_batch_tiles_test.sum()
print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
fpath = test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
disp_var = disp_var_test, # difference between maximal and minimal disparity for each scene, each tile
disp_neibs = num_neibs_test, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
num_gt_test = num_batch_tiles_test.sum()
high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test))
fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
plt.show()
scene = os.path.basename(test_corr)[:17]
scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
fname =scene+'-'+scene_version
img_filenameTFR = os.path.join(pathTFR,'img',fname)
print_time("Saving test image %s as tiles..."%(img_filenameTFR),end = " ")
writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
print_time("Done")
pass
pass
nn_ds_neibs2.py
View file @ 464cacc9
...@@ -31,7 +31,7 @@ FILES_PER_SCENE = 5 # number of random offset files for the scene to select f ...@@ -31,7 +31,7 @@ FILES_PER_SCENE = 5 # number of random offset files for the scene to select f
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from #MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch #MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500 #MAX_EPOCH = 500
LR = 1e-4 # learning rate LR = 1e-3 # learning rate
LR100 = 1e-4 LR100 = 1e-4
USE_CONFIDENCE = False USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False ABSOLUTE_DISPARITY = False # True # False # True # False
...@@ -47,8 +47,8 @@ RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batche ...@@ -47,8 +47,8 @@ RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batche
#BATCH_SIZE = 1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches #BATCH_SIZE = 1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True SHUFFLE_EPOCH = True
NET_ARCH1 = 0 # 6 #0 # 4 # 3 # overwrite with argv? NET_ARCH1 = 0 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 0 # 6 # 0 # 3 # overwrite with argv? NET_ARCH2 = 3 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation) ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
...@@ -331,7 +331,7 @@ except IndexError: ...@@ -331,7 +331,7 @@ except IndexError:
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords" train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
""" """
files_img = ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords'] files_img = ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords']
result_file = '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R.npy' result_file = '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-3.npy'
files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords", files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords", "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords",
...@@ -364,11 +364,11 @@ import tensorflow as tf ...@@ -364,11 +364,11 @@ import tensorflow as tf
import tensorflow.contrib.slim as slim import tensorflow.contrib.slim as slim
#try: try:
eval_results(result_file, ABSOLUTE_DISPARITY) eval_results(result_file, ABSOLUTE_DISPARITY)
exit(0) #exit(0)
#except: except:
# pass pass
datasets_img = [] datasets_img = []
for fpath in files_img: for fpath in files_img:
...@@ -503,7 +503,7 @@ dataset_img_size //= BATCH_SIZE ...@@ -503,7 +503,7 @@ dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size)) #print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE) dataset_tt = dataset_tt.batch(BATCH_SIZE)
#dataset_tt = dataset_tt.prefetch(BATCH_SIZE) dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator() iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next() next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size)) #print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
......
nn_ds_neibs3.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = 1e-4
LR200 = 3e-5
LR400 = 1e-5
LR600 = 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 1000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 1 # 1 - 3x3, 2 - 5x5 tiles
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
# target_disparity_list.append(np.array(example.features.feature['target_disparity'].float_list.value[0], dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
datasets_data = []
# for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
for nrec in range(len(datasets_lvar_data)):
rec1 = datasets_lvar_data[nrec]
rec2 = datasets_hvar_data[nrec]
rec = {'corr2d': np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
rec['corr2d'][0::2] = rec1['corr2d']
rec['corr2d'][1::2] = rec2['corr2d']
rec['target_disparity'][0::2] = rec1['target_disparity']
rec['target_disparity'][1::2] = rec2['target_disparity']
rec['gt_ds'][0::2] = rec1['gt_ds']
rec['gt_ds'][1::2] = rec2['gt_ds']
# if del_src:
# rec1['corr2d'] = None
# rec1['target_disparity'] = None
# rec1['gt_ds'] = None
# rec2['corr2d'] = None
# rec2['target_disparity'] = None
# rec2['gt_ds'] = None
if del_src:
datasets_lvar_data[nrec] = None
datasets_hvar_data[nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
files_img = ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords']
#result_file = '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-4.npy'
result_file = '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04'+SUFFIX+'.npy'
files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train002_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train003_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train004_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train005_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train006_R1_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train007_R1_LE_1.5.tfrecords",
]
files_train_hvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train002_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train003_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train004_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train005_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train006_R1_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_rand2/train007_R1_GT_1.5.tfrecords",
]
#files_train_hvar = []
#file_test_lvar= "/home/eyesis/x3d_data/data_sets/tf_data_3x3a/train000_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
file_test_lvar= "/home/eyesis/x3d_data/data_sets/tf_data_rand2/testTEST_R1_LE_1.5.tfrecords"
file_test_hvar= "/home/eyesis/x3d_data/data_sets/tf_data_rand2/testTEST_R1_GT_1.5.tfrecords" # None # "/home/eyesis/x3d_data/data_sets/tf_data_3x3/train-002_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
#file_test_hvar= None
weight_hvar = 0.13
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
try:
eval_results(result_file, ABSOLUTE_DISPARITY)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_train_hvar = []
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
if (file_test_lvar):
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
dataset_test_lvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
if (file_test_hvar):
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
dataset_test_hvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
if (file_test_lvar):
reduce_tile_size([dataset_test_lvar], TILE_LAYERS, TILE_SIDE)
if (file_test_hvar):
reduce_tile_size([dataset_test_hvar], TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
if (file_test_lvar):
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters([dataset_test_lvar])
print_time(" Done")
if (file_test_hvar):
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters([dataset_test_hvar])
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
if ZIP_LHVAR:
datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar)
pass
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
if (file_test_lvar):
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
if (file_test_hvar):
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(dataset_test_lvar['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
"""
iterator_test = dataset_test.make_initializable_iterator()
next_element_test = iterator_test.get_next()
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse | (j > 0)))
if num_non_sum > 0:
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
return fc[-1]
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?:9,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
inter_list.append(network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8))
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2)
#corr2d9x325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[None,cluster_size,FEATURES_PER_TILE]) , tf.reshape(next_element_tt['target_disparity'], [None,cluster_size, 1])],2)
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
#In GPU - reformat inputs
##corr2d325 = tf.concat([next_element_tt['corr2d'], next_element_tt['target_disparity']],1)
#Should have shape (?,9,325)
corr2d9x325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE]) , tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1])],2)
#corr2d9x324 = tf.reshape( next_element_tt['corr2d'], [-1, cluster_size, FEATURES_PER_TILE], name = 'corr2d9x324')
#td9x1 = tf.reshape(next_element_tt['target_disparity'], [-1, cluster_size, 1], name = 'td9x1')
#corr2d9x325 = tf.concat([corr2d9x324 , td9x1],2, name = 'corr2d9x325')
# in_features = tf.concat([corr2d,target_disparity],0)
#out = network_fc_simple(input=corr2d325, arch = NET_ARCH1)
out = network_siam(input=corr2d9x325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs3_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_avg = 0.0
train2_avg = 0.0
test_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
# num_train_variants = len(files_train_lvar)
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# learning_rate = [LR,LR100][epoch >=100]
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
# disp_out= np.empty((dataset_train_size * BATCH_SIZE), dtype=np.float32)
for i in range(dataset_train_size):
try:
# train_summary,_, G_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, corr2d325_out = sess.run(
train_summary,_, G_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
# corr2d325,
],
feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
# save all for now as a test
#train_writer.add_summary(summary, i)
#train_writer.add_summary(train_summary, i)
# disp_out[BATCH_SIZE*i:BATCH_SIZE*(i+1)] = output.flatten()
loss_train_hist[i] = G_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_avg = np.average(loss_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
for i in range(dataset_test_size):
try:
# test_summary, G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_test_hist[i] = G_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_avg = np.average(loss_test_hist).astype(np.float32)
test2_avg = np.average(loss2_test_hist).astype(np.float32)
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
# _,_=sess.run([tf_ph_G_loss,tf_ph_sq_diff],feed_dict={tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg})
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f (%f %f) dbg:%f %f"%(epoch,i,train_avg, test_avg,train2_avg, test2_avg, gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
# start_offs=0
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
# loss_test_hist[i] = G_loss_tested
# loss2_test_hist[i] = out_cost1
# gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
# disp_out[BATCH_SIZE*i:BATCH_SIZE*(i+1)] = output.flatten()
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
nn_ds_neibs4.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
import sys
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = 1e-4
LR200 = 3e-5
LR400 = 1e-5
LR600 = 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 1000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
#EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
BATCH_SIZE = 2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 2 # 1 # 1 - 3x3, 2 - 5x5 tiles
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
datasets_data = []
# for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
for nrec in range(len(datasets_lvar_data)):
rec1 = datasets_lvar_data[nrec]
rec2 = datasets_hvar_data[nrec]
rec = {'corr2d': np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
rec['corr2d'][0::2] = rec1['corr2d']
rec['corr2d'][1::2] = rec2['corr2d']
rec['target_disparity'][0::2] = rec1['target_disparity']
rec['target_disparity'][1::2] = rec2['target_disparity']
rec['gt_ds'][0::2] = rec1['gt_ds']
rec['gt_ds'][1::2] = rec2['gt_ds']
# if del_src:
# rec1['corr2d'] = None
# rec1['target_disparity'] = None
# rec1['gt_ds'] = None
# rec2['corr2d'] = None
# rec2['target_disparity'] = None
# rec2['gt_ds'] = None
if del_src:
datasets_lvar_data[nrec] = None
datasets_hvar_data[nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
files_img = ['/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/img/1527257933_150165-v04.tfrecords']
#result_file = '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-4.npy'
result_file = '/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/rslt/1527257933_150165-v04'+SUFFIX+'.npy'
files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train002_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train003_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train004_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train005_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train006_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train007_R2_LE_1.5.tfrecords",
]
files_train_hvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords", # replace with GT when ready
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
]
#files_train_hvar = []
#file_test_lvar= "/home/eyesis/x3d_data/data_sets/tf_data_3x3a/train000_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
#replace when available
file_test_lvar= "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_LE_1.5.tfrecords"
file_test_hvar= "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords" # None # "/home/eyesis/x3d_data/data_sets/tf_data_3x3/train-002_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
#file_test_hvar= None
weight_hvar = 0.13
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
try:
eval_results(result_file, ABSOLUTE_DISPARITY)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d": corr2d,
"target_disparity": target_disparity,
"gt_ds": gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_train_hvar = []
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
if (file_test_lvar):
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
dataset_test_lvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
if (file_test_hvar):
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
dataset_test_hvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
if (file_test_lvar):
reduce_tile_size([dataset_test_lvar], TILE_LAYERS, TILE_SIDE)
if (file_test_hvar):
reduce_tile_size([dataset_test_hvar], TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
if (file_test_lvar):
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters([dataset_test_lvar])
print_time(" Done")
if (file_test_hvar):
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters([dataset_test_hvar])
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
if ZIP_LHVAR:
print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar)
print_time(" Done")
pass
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
if (file_test_lvar):
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
if (file_test_hvar):
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
#dataset_tt TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(dataset_test_lvar['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
"""
next_element_tt dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}
'corr2d' (140405473715624) Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)
'gt_ds' (140405473715680) Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)
'target_disparity' (140405501995888) Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse | (j > 0)))
if num_non_sum > 0:
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
return fc[-1]
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?,9,325]-> [?,25,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
inter_list.append(network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8))
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2)
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
out = network_siam(input=corr2d_Nx325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs4_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_avg = 0.0
train2_avg = 0.0
test_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
# num_train_variants = len(files_train_lvar)
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# print ("sr1",file=sys.stderr,end=" ")
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
for i in range(dataset_train_size):
try:
train_summary,_, G_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_train_hist[i] = G_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_avg = np.average(loss_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
tst_avg = [0.0]*dataset_test_size
tst2_avg = [0.0]*dataset_test_size
for i in range(dataset_test_size):
try:
test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_test_hist[i] = G_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_avg = np.average(loss_test_hist).astype(np.float32)
tst_avg[i] = test_avg
test2_avg = np.average(loss2_test_hist).astype(np.float32)
tst2_avg[i] = test2_avg
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
nn_ds_neibs5.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
import sys
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = 1e-4
LR200 = 1e-4 # 3e-5
LR400 = 1e-4 # 1e-5
LR600 = 1e-4 # 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 1000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
#EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
BATCH_SIZE = 2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 2 # 1 # 1 - 3x3, 2 - 5x5 tiles
SHUFFLE_FILES = True
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
datasets_data = []
permut_l = np.random.permutation(len(datasets_lvar_data))
permut_h = np.random.permutation(len(datasets_hvar_data))
"""
TODO: add shuffling batches in each record
"""
# for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
for nrec in range(len(datasets_lvar_data)):
if shuffle:
rec1 = datasets_lvar_data[permut_l[nrec]]
rec2 = datasets_hvar_data[permut_h[nrec]]
else:
rec1 = datasets_lvar_data[nrec]
rec2 = datasets_hvar_data[nrec]
# print ("\n",nrec, permut_l[nrec], permut_h[nrec])
# print (rec1['corr2d'].shape,rec2['corr2d'].shape)
rec = {'corr2d': np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
rec['corr2d'][0::2] = rec1['corr2d']
rec['corr2d'][1::2] = rec2['corr2d']
rec['target_disparity'][0::2] = rec1['target_disparity']
rec['target_disparity'][1::2] = rec2['target_disparity']
rec['gt_ds'][0::2] = rec1['gt_ds']
rec['gt_ds'][1::2] = rec2['gt_ds']
if del_src:
datasets_lvar_data[nrec] = None
datasets_hvar_data[nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0,
radius = 0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool)
not_nan_ext[radius:-radius,radius:-radius] = not_nan
for dy in range(2*radius+1):
for dx in range(2*radius+1):
not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
not_nan = not_nan_ext[radius:-radius,radius:-radius]
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
files_img = ['/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/img/1527257933_150165-v04.tfrecords']
#result_file = '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-4.npy'
result_file = '/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/rslt/1527257933_150165-v04'+SUFFIX+'.npy'
files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train002_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train003_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train004_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train005_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train006_R2_LE_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train007_R2_LE_1.5.tfrecords",
]
files_train_hvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords", # replace with GT when ready
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train002_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train003_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train004_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train005_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train006_R2_GT_1.5.tfrecords",
"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
]
#files_train_hvar = []
#file_test_lvar= "/home/eyesis/x3d_data/data_sets/tf_data_3x3a/train000_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
#replace when available
file_test_lvar= "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/testTEST_R2_LE_1.5.tfrecords" #train000_R2_LE_1.5.tfrecords"
file_test_hvar= "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/testTEST_R2_GT_1.5.tfrecords" #train002_R2_GT_1.5.tfrecords" # None # "/home/eyesis/x3d_data/data_sets/tf_data_3x3/train-002_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
#file_test_hvar= None
weight_hvar = 0.13
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
try:
eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d": corr2d,
"target_disparity": target_disparity,
"gt_ds": gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_train_hvar = []
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
if (file_test_lvar):
print_time("Importing test data (low variance) from "+file_test_lvar, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
dataset_test_lvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
if (file_test_hvar):
print_time("Importing test data (high variance) from "+file_test_hvar, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
dataset_test_hvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
if (file_test_lvar):
reduce_tile_size([dataset_test_lvar], TILE_LAYERS, TILE_SIDE)
if (file_test_hvar):
reduce_tile_size([dataset_test_hvar], TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
if (file_test_lvar):
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters([dataset_test_lvar])
print_time(" Done")
if (file_test_hvar):
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters([dataset_test_hvar])
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
if ZIP_LHVAR:
print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
print_time(" Done")
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
if (file_test_lvar):
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
if (file_test_hvar):
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
#dataset_tt TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(dataset_test_lvar['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
"""
next_element_tt dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}
'corr2d' (140405473715624) Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)
'gt_ds' (140405473715680) Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)
'target_disparity' (140405501995888) Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse | (j > 0)))
if num_non_sum > 0:
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
return fc[-1]
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?,9,325]-> [?,25,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
inter_list.append(network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8))
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2)
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
out = network_siam(input=corr2d_Nx325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs5_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_avg = 0.0
train2_avg = 0.0
test_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
# num_train_variants = len(files_train_lvar)
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# print ("sr1",file=sys.stderr,end=" ")
if (file_index == 0) and SHUFFLE_FILES:
print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
print_time(" Done")
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
for i in range(dataset_train_size):
try:
train_summary,_, G_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_train_hist[i] = G_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_avg = np.average(loss_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
tst_avg = [0.0]*len(datasets_test)
tst2_avg = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
for i in range(dataset_test_size):
try:
test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_test_hist[i] = G_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_avg = np.average(loss_test_hist).astype(np.float32)
tst_avg[ntest] = test_avg
test2_avg = np.average(loss2_test_hist).astype(np.float32)
tst2_avg[ntest] = test2_avg
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
nn_ds_neibs6.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
import sys
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = LR # 1e-4
LR200 = LR100 # 3e-5
LR400 = LR200 # 1e-5
LR600 = LR400 # 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 3000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
#EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
BATCH_SIZE = 2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 0 #3 # 0 # 0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 0 # 0 # 3 # 0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 2 # 1 # 1 - 3x3, 2 - 5x5 tiles
SHUFFLE_FILES = True
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def permute_to_swaps(perm):
pairs = []
for i in range(len(perm)):
w = np.where(perm == i)[0][0]
if w != i:
pairs.append([i,w])
perm[w] = perm[i]
perm[i] = i
return pairs
def shuffle_in_place(datasets_data, indx, period):
swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
num_entries = datasets_data[0]['corr2d'].shape[0] // period
for swp in swaps:
ds0 = datasets_data[swp[0]]
ds1 = datasets_data[swp[1]]
tmp = ds0['corr2d'][indx::period].copy()
ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
ds1['corr2d'][indx::period] = tmp
tmp = ds0['target_disparity'][indx::period].copy()
ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
ds1['target_disparity'][indx::period] = tmp
tmp = ds0['gt_ds'][indx::period].copy()
ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
ds1['gt_ds'][indx::period] = tmp
def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
datasets_data = []
permut_l = np.random.permutation(len(datasets_lvar_data))
permut_h = np.random.permutation(len(datasets_hvar_data))
"""
TODO: add shuffling batches in each record
"""
# for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
for nrec in range(len(datasets_lvar_data)):
if shuffle:
rec1 = datasets_lvar_data[permut_l[nrec]]
rec2 = datasets_hvar_data[permut_h[nrec]]
else:
rec1 = datasets_lvar_data[nrec]
rec2 = datasets_hvar_data[nrec]
# print ("\n",nrec, permut_l[nrec], permut_h[nrec])
# print (rec1['corr2d'].shape,rec2['corr2d'].shape)
rec = {'corr2d': np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
rec['corr2d'][0::2] = rec1['corr2d']
rec['corr2d'][1::2] = rec2['corr2d']
rec['target_disparity'][0::2] = rec1['target_disparity']
rec['target_disparity'][1::2] = rec2['target_disparity']
rec['gt_ds'][0::2] = rec1['gt_ds']
rec['gt_ds'][1::2] = rec2['gt_ds']
if del_src:
datasets_lvar_data[nrec] = None
datasets_hvar_data[nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0,
radius = 0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool)
not_nan_ext[radius:-radius,radius:-radius] = not_nan
for dy in range(2*radius+1):
for dx in range(2*radius+1):
not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
not_nan = not_nan_ext[radius:-radius,radius:-radius]
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
data_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
img_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
dir_train_lvar = data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar = data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_lvar = data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_hvar = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_img = os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
dir_result = os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
"""
Try again - all hvar training, train than different hvar testing.
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
match for either of hvar/lvar, even those that were used for training
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
try same with higher LR - will they eventually converge?
Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
"""
#just testing:
#files_train_lvar = files_train_hvar
files_test_lvar = ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
files_test_hvar = ["train004_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
#files_img = ['1527257933_150165-v04'] # overlook
#files_img = ['1527256858_150165-v01'] # state Street
files_img = ['1527256816_150165-v02'] # staete Street
for i, path in enumerate(files_train_lvar):
files_train_lvar[i]=os.path.join(dir_train_lvar, path)
for i, path in enumerate(files_train_hvar):
files_train_hvar[i]=os.path.join(dir_train_hvar, path)
for i, path in enumerate(files_test_lvar):
files_test_lvar[i]=os.path.join(dir_test_lvar, path)
for i, path in enumerate(files_test_hvar):
files_test_hvar[i]=os.path.join(dir_test_hvar, path)
result_files=[]
for i, path in enumerate(files_img):
files_img[i] = os.path.join(dir_img, path+'.tfrecords')
result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
#file_test_hvar= None
weight_hvar = 0.26
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
for result_file in result_files:
try:
eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d": corr2d,
"target_disparity": target_disparity,
"gt_ds": gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_train_hvar = []
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
"""
if (file_test_lvar):
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
dataset_test_lvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
if (file_test_hvar):
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
dataset_test_hvar = {"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds}
print_time(" Done")
"""
datasets_test_lvar = []
for fpath in files_test_lvar:
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_hvar = []
for fpath in files_test_hvar:
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_hvar, TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters(datasets_test_lvar)
print_time(" Done")
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters(datasets_test_hvar)
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
if ZIP_LHVAR:
print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
# datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar) # no shuffle, delete src
print_time(" Done")
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
for dataset_test_lvar in datasets_test_lvar:
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
for dataset_test_hvar in datasets_test_hvar:
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
#dataset_tt TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
"""
next_element_tt dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}
'corr2d' (140405473715624) Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)
'gt_ds' (140405473715680) Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)
'target_disparity' (140405501995888) Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse | (j > 0)))
if num_non_sum > 0:
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
return fc[-1]
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?,9,325]-> [?,25,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
inter_list.append(network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8))
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2)
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
out = network_siam(input=corr2d_Nx325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs6_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_avg = 0.0
train2_avg = 0.0
test_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
# num_train_variants = len(files_train_lvar)
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# print ("sr1",file=sys.stderr,end=" ")
if (file_index == 0) and SHUFFLE_FILES:
print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
# datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
shuffle_in_place (datasets_train, 0, 2)
shuffle_in_place (datasets_train, 1, 2)
print_time(" Done")
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
for i in range(dataset_train_size):
try:
train_summary,_, G_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_train_hist[i] = G_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_avg = np.average(loss_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
tst_avg = [0.0]*len(datasets_test)
tst2_avg = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
for i in range(dataset_test_size):
try:
test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_test_hist[i] = G_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_avg = np.average(loss_test_hist).astype(np.float32)
tst_avg[ntest] = test_avg
test2_avg = np.average(loss2_test_hist).astype(np.float32)
tst2_avg[ntest] = test2_avg
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
result_file = result_files[ntest]
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
nn_ds_neibs7.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
import sys
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = 3e-4 #LR # 1e-4
LR200 = 1e-4 #LR100 # 3e-5
LR400 = 3e-5 #LR200 # 1e-5
LR600 = 1e-5 #LR400 # 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 3000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
#EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
TWO_TRAINS = True # use 2 train sets
BATCH_SIZE = ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 4 # #4 # 8 # 4 # 0 #3 # 0 # 0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 4 # 0 # 0 # 4 # 0 # 0 # 3 # 0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = True # False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 2 # 1 # 1 - 3x3, 2 - 5x5 tiles
SHUFFLE_FILES = True
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def permute_to_swaps(perm):
pairs = []
for i in range(len(perm)):
w = np.where(perm == i)[0][0]
if w != i:
pairs.append([i,w])
perm[w] = perm[i]
perm[i] = i
return pairs
def shuffle_in_place(datasets_data, indx, period):
swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
num_entries = datasets_data[0]['corr2d'].shape[0] // period
for swp in swaps:
ds0 = datasets_data[swp[0]]
ds1 = datasets_data[swp[1]]
tmp = ds0['corr2d'][indx::period].copy()
ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
ds1['corr2d'][indx::period] = tmp
tmp = ds0['target_disparity'][indx::period].copy()
ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
ds1['target_disparity'][indx::period] = tmp
tmp = ds0['gt_ds'][indx::period].copy()
ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
ds1['gt_ds'][indx::period] = tmp
def shuffle_chunks_in_place(datasets_data, tiles_groups_per_chunk):
"""
Improve shuffling by preserving indices inside batches (0 <->0, ... 39 <->39 for 40 tile group batches)
"""
num_files = len(datasets_data)
#chunks_per_file = datasets_data[0]['target_disparity']
for nf, ds in enumerate(datasets_data):
groups_per_file = ds['corr2d'].shape[0]
chunks_per_file = groups_per_file//tiles_groups_per_chunk
permut = np.random.permutation(chunks_per_file)
ds['corr2d'] = ds['corr2d']. reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
ds['target_disparity'] = ds['target_disparity'].reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
ds['gt_ds'] = ds['gt_ds']. reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
def zip_lvar_hvar_old(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
datasets_data = []
permut_l = np.random.permutation(len(datasets_lvar_data))
permut_h = np.random.permutation(len(datasets_hvar_data))
"""
TODO: add shuffling batches in each record
"""
for nrec in range(len(datasets_lvar_data)):
if shuffle:
rec1 = datasets_lvar_data[permut_l[nrec]]
rec2 = datasets_hvar_data[permut_h[nrec]]
else:
rec1 = datasets_lvar_data[nrec]
rec2 = datasets_hvar_data[nrec]
# print ("\n",nrec, permut_l[nrec], permut_h[nrec])
# print (rec1['corr2d'].shape,rec2['corr2d'].shape)
rec = {'corr2d': np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
rec['corr2d'][0::2] = rec1['corr2d']
rec['corr2d'][1::2] = rec2['corr2d']
rec['target_disparity'][0::2] = rec1['target_disparity']
rec['target_disparity'][1::2] = rec2['target_disparity']
rec['gt_ds'][0::2] = rec1['gt_ds']
rec['gt_ds'][1::2] = rec2['gt_ds']
if del_src:
datasets_lvar_data[nrec] = None
datasets_hvar_data[nrec] = None
datasets_data.append(rec)
return datasets_data
def zip_lvar_hvar(datasets_all_data, del_src = True):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
num_sets_to_combine = len(datasets_all_data)
datasets_data = []
if num_sets_to_combine:
for nrec in range(len(datasets_all_data[0])):
recs = [[] for _ in range(num_sets_to_combine)]
for nset, datasets in enumerate(datasets_all_data):
recs[nset] = datasets[nrec]
rec = {'corr2d': np.empty((recs[0]['corr2d'].shape[0]*num_sets_to_combine, recs[0]['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((recs[0]['target_disparity'].shape[0]*num_sets_to_combine,recs[0]['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((recs[0]['gt_ds'].shape[0]*num_sets_to_combine, recs[0]['gt_ds'].shape[1]),dtype=np.float32)}
for nset, reci in enumerate(recs):
rec['corr2d'] [nset::num_sets_to_combine] = recs[nset]['corr2d']
rec['target_disparity'][nset::num_sets_to_combine] = recs[nset]['target_disparity']
rec['gt_ds'] [nset::num_sets_to_combine] = recs[nset]['gt_ds']
if del_src:
for nset in range(num_sets_to_combine):
datasets_all_data[nset][nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0,
radius = 0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool)
not_nan_ext[radius:-radius,radius:-radius] = not_nan
for dy in range(2*radius+1):
for dx in range(2*radius+1):
not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
not_nan = not_nan_ext[radius:-radius,radius:-radius]
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
data_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
data_dir1 = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
img_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
dir_train_lvar = data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar = data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_lvar1 = data_dir1 #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar1 = data_dir1 # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_lvar = data_dir # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_hvar = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_3" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_img = os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
dir_result = os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
files_train_lvar1 = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar1 = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
"""
Try again - all hvar training, train than different hvar testing.
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
match for either of hvar/lvar, even those that were used for training
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
try same with higher LR - will they eventually converge?
Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
"""
#just testing:
#files_train_lvar = files_train_hvar
files_test_lvar = ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
files_test_hvar = ["train000_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
#files_img = ['1527257933_150165-v04'] # overlook
#files_img = ['1527256858_150165-v01'] # state Street
files_img = ['1527256816_150165-v02'] # staete Street
for i, path in enumerate(files_train_lvar):
files_train_lvar[i]=os.path.join(dir_train_lvar, path)
for i, path in enumerate(files_train_hvar):
files_train_hvar[i]=os.path.join(dir_train_hvar, path)
# Second set of files
for i, path in enumerate(files_train_lvar1):
files_train_lvar1[i]=os.path.join(dir_train_lvar1, path)
for i, path in enumerate(files_train_hvar1):
files_train_hvar1[i]=os.path.join(dir_train_hvar1, path)
for i, path in enumerate(files_test_lvar):
files_test_lvar[i]=os.path.join(dir_test_lvar, path)
for i, path in enumerate(files_test_hvar):
files_test_hvar[i]=os.path.join(dir_test_hvar, path)
result_files=[]
for i, path in enumerate(files_img):
files_img[i] = os.path.join(dir_img, path+'.tfrecords')
result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
#file_test_hvar= None
weight_hvar = 0.26
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
for result_file in result_files:
try:
eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d": corr2d,
"target_disparity": target_disparity,
"gt_ds": gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
datasets_train_hvar = []
datasets_train_lvar1 = []
datasets_train_hvar1 = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
if TWO_TRAINS:
for fpath in files_train_lvar1:
print_time("Importing train data1 (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar1.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
for fpath in files_train_hvar1:
print_time("Importing train data1 (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar1.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_lvar = []
for fpath in files_test_lvar:
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_hvar = []
for fpath in files_test_hvar:
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_lvar1, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar1, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_hvar, TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
print_time("Reshaping train data1 (low variance)", end="")
reformat_to_clusters(datasets_train_lvar1)
print_time(" Done")
print_time("Reshaping train data1 (high variance)", end="")
reformat_to_clusters(datasets_train_hvar1)
print_time(" Done")
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters(datasets_test_lvar)
print_time(" Done")
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters(datasets_test_hvar)
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
datasets_train_list = [datasets_train_lvar, datasets_train_hvar]
if TWO_TRAINS:
datasets_train_list += [datasets_train_lvar1, datasets_train_hvar1]
if ZIP_LHVAR:
print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
datasets_train = zip_lvar_hvar(datasets_train_list, del_src = True) # no shuffle, delete src
print_time(" Done")
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
for dataset_test_lvar in datasets_test_lvar:
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
for dataset_test_hvar in datasets_test_hvar:
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
#dataset_tt TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
"""
next_element_tt dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}
'corr2d' (140405473715624) Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)
'gt_ds' (140405473715680) Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)
'target_disparity' (140405501995888) Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse | (j > 0)))
if num_non_sum > 0:
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
return fc[-1]
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?,9,325]-> [?,25,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
inter_list.append(network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8))
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2)
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
out = network_siam(input=corr2d_Nx325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs7_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_avg = 0.0
train2_avg = 0.0
test_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# print ("sr1",file=sys.stderr,end=" ")
if (file_index == 0) and SHUFFLE_FILES:
num_sets = len(datasets_train_list)
print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
for i in range(num_sets):
shuffle_in_place (datasets_train, i, num_sets)
print_time(" Done")
print_time("Shuffling tile chunks ", end="")
shuffle_chunks_in_place (datasets_train, 1)
print_time(" Done")
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
for i in range(dataset_train_size):
try:
train_summary,_, G_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_train_hist[i] = G_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_avg = np.average(loss_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
tst_avg = [0.0]*len(datasets_test)
tst2_avg = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
for i in range(dataset_test_size):
try:
test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_test_hist[i] = G_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_avg = np.average(loss_test_hist).astype(np.float32)
tst_avg[ntest] = test_avg
test2_avg = np.average(loss2_test_hist).astype(np.float32)
tst2_avg[ntest] = test2_avg
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
result_file = result_files[ntest]
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
nn_ds_neibs8.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
import sys
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = 3e-4 #LR # 1e-4
LR200 = 1e-4 #LR100 # 3e-5
LR400 = 3e-5 #LR200 # 1e-5
LR600 = 1e-5 #LR400 # 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 3000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
#EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
TWO_TRAINS = True # use 2 train sets
BATCH_SIZE = ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 0 # 4 # #4 # 8 # 4 # 0 #3 # 0 # 0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 0 # 4 # 0 # 0 # 4 # 0 # 0 # 3 # 0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = False # True # False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 2 # 1 # 1 - 3x3, 2 - 5x5 tiles
SHUFFLE_FILES = True
WLOSS_LAMBDA = 50.0 # 5.0 # 1.0 # fraction of the W_loss (input layers weight non-uniformity) added to G_loss
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])+"LMBD"+str(WLOSS_LAMBDA)
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def permute_to_swaps(perm):
pairs = []
for i in range(len(perm)):
w = np.where(perm == i)[0][0]
if w != i:
pairs.append([i,w])
perm[w] = perm[i]
perm[i] = i
return pairs
def shuffle_in_place(datasets_data, indx, period):
swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
num_entries = datasets_data[0]['corr2d'].shape[0] // period
for swp in swaps:
ds0 = datasets_data[swp[0]]
ds1 = datasets_data[swp[1]]
tmp = ds0['corr2d'][indx::period].copy()
ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
ds1['corr2d'][indx::period] = tmp
tmp = ds0['target_disparity'][indx::period].copy()
ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
ds1['target_disparity'][indx::period] = tmp
tmp = ds0['gt_ds'][indx::period].copy()
ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
ds1['gt_ds'][indx::period] = tmp
def shuffle_chunks_in_place(datasets_data, tiles_groups_per_chunk):
"""
Improve shuffling by preserving indices inside batches (0 <->0, ... 39 <->39 for 40 tile group batches)
"""
num_files = len(datasets_data)
#chunks_per_file = datasets_data[0]['target_disparity']
for nf, ds in enumerate(datasets_data):
groups_per_file = ds['corr2d'].shape[0]
chunks_per_file = groups_per_file//tiles_groups_per_chunk
permut = np.random.permutation(chunks_per_file)
ds['corr2d'] = ds['corr2d']. reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
ds['target_disparity'] = ds['target_disparity'].reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
ds['gt_ds'] = ds['gt_ds']. reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
def zip_lvar_hvar_old(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
datasets_data = []
permut_l = np.random.permutation(len(datasets_lvar_data))
permut_h = np.random.permutation(len(datasets_hvar_data))
"""
TODO: add shuffling batches in each record
"""
for nrec in range(len(datasets_lvar_data)):
if shuffle:
rec1 = datasets_lvar_data[permut_l[nrec]]
rec2 = datasets_hvar_data[permut_h[nrec]]
else:
rec1 = datasets_lvar_data[nrec]
rec2 = datasets_hvar_data[nrec]
# print ("\n",nrec, permut_l[nrec], permut_h[nrec])
# print (rec1['corr2d'].shape,rec2['corr2d'].shape)
rec = {'corr2d': np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
rec['corr2d'][0::2] = rec1['corr2d']
rec['corr2d'][1::2] = rec2['corr2d']
rec['target_disparity'][0::2] = rec1['target_disparity']
rec['target_disparity'][1::2] = rec2['target_disparity']
rec['gt_ds'][0::2] = rec1['gt_ds']
rec['gt_ds'][1::2] = rec2['gt_ds']
if del_src:
datasets_lvar_data[nrec] = None
datasets_hvar_data[nrec] = None
datasets_data.append(rec)
return datasets_data
def zip_lvar_hvar(datasets_all_data, del_src = True):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
num_sets_to_combine = len(datasets_all_data)
datasets_data = []
if num_sets_to_combine:
for nrec in range(len(datasets_all_data[0])):
recs = [[] for _ in range(num_sets_to_combine)]
for nset, datasets in enumerate(datasets_all_data):
recs[nset] = datasets[nrec]
rec = {'corr2d': np.empty((recs[0]['corr2d'].shape[0]*num_sets_to_combine, recs[0]['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((recs[0]['target_disparity'].shape[0]*num_sets_to_combine,recs[0]['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((recs[0]['gt_ds'].shape[0]*num_sets_to_combine, recs[0]['gt_ds'].shape[1]),dtype=np.float32)}
for nset, reci in enumerate(recs):
rec['corr2d'] [nset::num_sets_to_combine] = recs[nset]['corr2d']
rec['target_disparity'][nset::num_sets_to_combine] = recs[nset]['target_disparity']
rec['gt_ds'] [nset::num_sets_to_combine] = recs[nset]['gt_ds']
if del_src:
for nset in range(num_sets_to_combine):
datasets_all_data[nset][nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0,
radius = 0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool)
not_nan_ext[radius:-radius,radius:-radius] = not_nan
for dy in range(2*radius+1):
for dx in range(2*radius+1):
not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
not_nan = not_nan_ext[radius:-radius,radius:-radius]
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
data_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
data_dir1 = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
img_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
dir_train_lvar = data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar = data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_lvar1 = data_dir1 #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar1 = data_dir1 # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_lvar = data_dir # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_hvar = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_3" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_img = os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
dir_result = os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
files_train_lvar1 = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar1 = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
"""
Try again - all hvar training, train than different hvar testing.
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
match for either of hvar/lvar, even those that were used for training
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
try same with higher LR - will they eventually converge?
Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
"""
#just testing:
#files_train_lvar = files_train_hvar
files_test_lvar = ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
files_test_hvar = ["train000_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
#files_img = ['1527257933_150165-v04'] # overlook
#files_img = ['1527256858_150165-v01'] # state Street
files_img = ['1527256816_150165-v02'] # staete Street
for i, path in enumerate(files_train_lvar):
files_train_lvar[i]=os.path.join(dir_train_lvar, path)
for i, path in enumerate(files_train_hvar):
files_train_hvar[i]=os.path.join(dir_train_hvar, path)
# Second set of files
for i, path in enumerate(files_train_lvar1):
files_train_lvar1[i]=os.path.join(dir_train_lvar1, path)
for i, path in enumerate(files_train_hvar1):
files_train_hvar1[i]=os.path.join(dir_train_hvar1, path)
for i, path in enumerate(files_test_lvar):
files_test_lvar[i]=os.path.join(dir_test_lvar, path)
for i, path in enumerate(files_test_hvar):
files_test_hvar[i]=os.path.join(dir_test_hvar, path)
result_files=[]
for i, path in enumerate(files_img):
files_img[i] = os.path.join(dir_img, path+'.tfrecords')
result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
#file_test_hvar= None
weight_hvar = 0.26
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
for result_file in result_files:
try:
eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d": corr2d,
"target_disparity": target_disparity,
"gt_ds": gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
datasets_train_hvar = []
datasets_train_lvar1 = []
datasets_train_hvar1 = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
if TWO_TRAINS:
for fpath in files_train_lvar1:
print_time("Importing train data1 (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar1.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
for fpath in files_train_hvar1:
print_time("Importing train data1 (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar1.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_lvar = []
for fpath in files_test_lvar:
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_hvar = []
for fpath in files_test_hvar:
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_lvar1, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar1, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_hvar, TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
print_time("Reshaping train data1 (low variance)", end="")
reformat_to_clusters(datasets_train_lvar1)
print_time(" Done")
print_time("Reshaping train data1 (high variance)", end="")
reformat_to_clusters(datasets_train_hvar1)
print_time(" Done")
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters(datasets_test_lvar)
print_time(" Done")
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters(datasets_test_hvar)
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
datasets_train_list = [datasets_train_lvar, datasets_train_hvar]
if TWO_TRAINS:
datasets_train_list += [datasets_train_lvar1, datasets_train_hvar1]
if ZIP_LHVAR:
print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
datasets_train = zip_lvar_hvar(datasets_train_list, del_src = True) # no shuffle, delete src
print_time(" Done")
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
for dataset_test_lvar in datasets_test_lvar:
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
for dataset_test_hvar in datasets_test_hvar:
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
#dataset_tt TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
"""
next_element_tt dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}
'corr2d' (140405473715624) Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)
'gt_ds' (140405473715680) Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)
'target_disparity' (140405501995888) Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
inp_weights = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
reuse_this = reuse | (j > 0)
scp = 'g_fc_sub'+str(i)
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope= scp, reuse = reuse_this))
if not reuse_this:
with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
if num_non_sum > 0:
reuse_this = reuse
scp = 'g_fc_sub'+str(i)+"r"
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope=scp, reuse = reuse_this))
if not reuse_this:
with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
scp = 'g_fc_sub'+str(i)
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope= scp, reuse = reuse))
if not reuse:
with tf.variable_scope(scp, reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
return fc[-1], inp_weights
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?,9,325]-> [?,25,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
inp_weights = []
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
# inter_list.append(network_sub(input[:,i,:],
# layout= layout1,
# reuse= reuse,
# sym8 = sym8))
ns, ns_weights = network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8)
inter_list.append(ns)
inp_weights += ns_weights
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2), inp_weights
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
def weightsLoss(inp_weights): # [batch_size,(1..2)] tf_result
# weights_lambdas): # single lambda or same length as inp_weights.shape[1]
"""
Enforcing 'smooth' weights for the input 2d correlation tiles
@return mean squared difference for each weight and average of 8 neighbors divided by mean squared weights
"""
weight_ortho = 1.0
weight_diag = 0.7
sw = 4.0 * (weight_ortho + weight_diag)
weight_ortho /= sw
weight_diag /= sw
# w_neib = tf.const([[weight_diag, weight_ortho, weight_diag],
# [weight_ortho, -1.0, weight_ortho],
# [weight_diag, weight_ortho, weight_diag]])
with tf.name_scope("WeightsLoss"):
# Adding 1 tile border
tf_inp = tf.reshape(inp_weights[:TILE_LAYERS * TILE_SIZE,:], [TILE_LAYERS, FILE_TILE_SIDE, FILE_TILE_SIDE, inp_weights.shape[1]], name = "tf_inp")
tf_inp_ext_h = tf.concat([tf_inp [:, :, :1, :], tf_inp, tf_inp [:, :, -1:, :]], axis = 2, name ="tf_inp_ext_h")
tf_inp_ext = tf.concat([tf_inp_ext_h [:, :1, :, :], tf_inp_ext_h, tf_inp_ext_h[:, -1:, :, :]], axis = 1, name ="tf_inp_ext")
s_ortho = tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:,:] + tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:, :]
s_corn = tf_inp_ext[:, :-2,:-2,:] + tf_inp_ext[:, :-2, 2:,:] + tf_inp_ext[:,2:, :-2,:] + tf_inp_ext[:,2: , 2:, :]
w_diff = tf.subtract(tf_inp, s_ortho * weight_ortho + s_corn * weight_diag, name="w_diff")
w_diff2 = tf.multiply(w_diff, w_diff, name="w_diff2")
w_var = tf.reduce_mean(w_diff2, name="w_var")
w2_mean = tf.reduce_mean(inp_weights * inp_weights, name="w2_mean")
w_rel = tf.divide(w_var, w2_mean, name= "w_rel")
return w_rel # scalar, cost for weights non-smoothness in 2d
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
out, inp_weights = network_siam(input=corr2d_Nx325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
if WLOSS_LAMBDA > 0.0:
W_loss = weightsLoss(inp_weights[0]) # inp_weights - list of tensors, currently - just [0]
GW_loss = tf.add(G_loss, WLOSS_LAMBDA * W_loss, name = "GW_loss")
else:
GW_loss = G_loss
W_loss = 0.0
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_W_loss = tf.placeholder(tf.float32,shape=None,name='W_loss_avg')
tf_ph_GW_loss = tf.placeholder(tf.float32,shape=None,name='GW_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("GW_loss", GW_loss)
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("W_loss", W_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("GW_loss_epoch", tf_ph_GW_loss)
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("W_loss_epoch", tf_ph_W_loss)
tf.summary.scalar("sq_diff_epoch", tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(GW_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs8_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_gw_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_g_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_w_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_gw_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_g_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_w_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_gw_avg = 0.0
train_g_avg = 0.0
train_w_avg = 0.0
test_gw_avg = 0.0
test_g_avg = 0.0
test_w_avg = 0.0
train2_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# print ("sr1",file=sys.stderr,end=" ")
if (file_index == 0) and SHUFFLE_FILES:
num_sets = len(datasets_train_list)
print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
for i in range(num_sets):
shuffle_in_place (datasets_train, i, num_sets)
print_time(" Done")
print_time("Shuffling tile chunks ", end="")
shuffle_chunks_in_place (datasets_train, 1)
print_time(" Done")
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
for i in range(dataset_train_size):
try:
train_summary,_, GW_loss_trained, G_loss_trained, W_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
GW_loss,
G_loss,
W_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr: learning_rate,
tf_ph_GW_loss: train_gw_avg,
tf_ph_G_loss: train_g_avg,
tf_ph_W_loss: train_w_avg,
tf_ph_sq_diff: train2_avg,
tf_gtvar_diff: gtvar_train_avg,
tf_img_test0: img_gain_test0,
tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_gw_train_hist[i] = GW_loss_trained
loss_g_train_hist[i] = G_loss_trained
loss_w_train_hist[i] = W_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_gw_avg = np.average(loss_gw_train_hist).astype(np.float32)
train_g_avg = np.average(loss_g_train_hist).astype(np.float32)
train_w_avg = np.average(loss_w_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
tst_avg = [0.0]*len(datasets_test)
tst2_avg = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
for i in range(dataset_test_size):
try:
test_summaries[ntest], GW_loss_tested, G_loss_tested, W_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
GW_loss,
G_loss,
W_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr: learning_rate,
tf_ph_GW_loss: test_gw_avg,
tf_ph_G_loss: test_g_avg,
tf_ph_W_loss: test_w_avg,
tf_ph_sq_diff: test2_avg,
tf_gtvar_diff: gtvar_test_avg,
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9}) # previous value of *_avg
loss_gw_test_hist[i] = GW_loss_tested
loss_g_test_hist[i] = G_loss_tested
loss_w_test_hist[i] = W_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_gw_avg = np.average(loss_gw_test_hist).astype(np.float32)
test_g_avg = np.average(loss_g_test_hist).astype(np.float32)
test_w_avg = np.average(loss_w_test_hist).astype(np.float32)
tst_avg[ntest] = test_gw_avg
test2_avg = np.average(loss2_test_hist).astype(np.float32)
tst2_avg[ntest] = test2_avg
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_gw_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr: learning_rate,
tf_ph_GW_loss: test_gw_avg,
tf_ph_G_loss: test_g_avg,
tf_ph_W_loss: test_w_avg,
tf_ph_sq_diff: test2_avg,
tf_gtvar_diff: gtvar_test_avg,
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9}) # previous value of *_avg
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
result_file = result_files[ntest]
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
nn_ds_neibs9.py 0 → 100644
View file @ 464cacc9
#!/usr/bin/env python3
from numpy import float64
from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
from debian.deb822 import PdiffIndex
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
from PIL import Image
import os
import sys
import glob
import numpy as np
import itertools
import time
import matplotlib.pyplot as plt
import shutil
import sys
TIME_START = time.time()
TIME_LAST = TIME_START
DEBUG_LEVEL= 1
DISP_BATCH_BINS = 20 # Number of batch disparity bins
STR_BATCH_BINS = 10 # Number of batch strength bins
FILES_PER_SCENE = 5 # number of random offset files for the scene to select from (0 - use all available)
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from
#MAX_BATCH_FILES = 10 #maximal number of files to use in a batch
#MAX_EPOCH = 500
LR = 1e-3 # learning rate
LR100 = 3e-4 #LR # 1e-4
LR200 = 1e-4 #LR100 # 3e-5
LR400 = 3e-5 #LR200 # 1e-5
LR600 = 1e-5 #LR400 # 3e-6
USE_CONFIDENCE = False
ABSOLUTE_DISPARITY = False # True # False # True # False
DEBUG_PLT_LOSS = True
TILE_LAYERS = 4
FILE_TILE_SIDE = 9
TILE_SIDE = 9 # 7
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
EPOCHS_TO_RUN = 3000#0 #0
EPOCHS_FULL_TEST = 5 # 10 # 25# repeat full image test after this number of epochs
#EPOCHS_SAME_FILE = 20
RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batches
#BATCH_SIZE = 2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
TWO_TRAINS = True # use 2 train sets
BATCH_SIZE = ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SHUFFLE_EPOCH = True
NET_ARCH1 = 0 # 4 # #4 # 8 # 4 # 0 #3 # 0 # 0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
NET_ARCH2 = 0 # 4 # 0 # 0 # 4 # 0 # 0 # 3 # 0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
SYM8_SUB = True #False # True # False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
ONLY_TILE = None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
ZIP_LHVAR = True # combine _lvar and _hvar as odd/even elements
#DEBUG_PACK_TILES = True
# CLUSTER_RADIUS should match input data
CLUSTER_RADIUS = 2 # 1 # 1 - 3x3, 2 - 5x5 tiles
SHUFFLE_FILES = True
WLOSS_LAMBDA = 50.0 # 5.0 # 1.0 # fraction of the W_loss (input layers weight non-uniformity) added to G_loss
SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])+"LMBD"+str(WLOSS_LAMBDA)
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
}
NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
#reading to memory (testing)
def readTFRewcordsEpoch(train_filename):
# filenames = [train_filename]
# dataset = tf.data.TFRecorDataset(filenames)
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, file_extension = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_corr2d = os.path.join(npy_dir,filebasename + '_corr2d.npy')
file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
file_gt_ds = os.path.join(npy_dir,filebasename + '_gt_ds.npy')
if (os.path.exists(file_corr2d) and
os.path.exists(file_target_disparity) and
os.path.exists(file_gt_ds)):
corr2d= np.load (file_corr2d)
target_disparity = np.load(file_target_disparity)
gt_ds = np.load(file_gt_ds)
pass
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
try:
os.makedirs(os.path.dirname(file_corr2d))
except:
pass
np.save(file_corr2d, corr2d)
np.save(file_target_disparity, target_disparity)
np.save(file_gt_ds, gt_ds)
return corr2d, target_disparity, gt_ds
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
# still some nan-s in correlation data?
# in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)
# corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius):
side = 2 * radius + 1
size = side * side
width = 324
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
pass
def reformat_to_clusters(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
def replace_nan(datasets_data):
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
for rec in datasets_data:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
np.nan_to_num(rec['gt_ds'], copy = False)
def permute_to_swaps(perm):
pairs = []
for i in range(len(perm)):
w = np.where(perm == i)[0][0]
if w != i:
pairs.append([i,w])
perm[w] = perm[i]
perm[i] = i
return pairs
def shuffle_in_place(datasets_data, indx, period):
swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
num_entries = datasets_data[0]['corr2d'].shape[0] // period
for swp in swaps:
ds0 = datasets_data[swp[0]]
ds1 = datasets_data[swp[1]]
tmp = ds0['corr2d'][indx::period].copy()
ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
ds1['corr2d'][indx::period] = tmp
tmp = ds0['target_disparity'][indx::period].copy()
ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
ds1['target_disparity'][indx::period] = tmp
tmp = ds0['gt_ds'][indx::period].copy()
ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
ds1['gt_ds'][indx::period] = tmp
def shuffle_chunks_in_place(datasets_data, tiles_groups_per_chunk):
"""
Improve shuffling by preserving indices inside batches (0 <->0, ... 39 <->39 for 40 tile group batches)
"""
num_files = len(datasets_data)
#chunks_per_file = datasets_data[0]['target_disparity']
for nf, ds in enumerate(datasets_data):
groups_per_file = ds['corr2d'].shape[0]
chunks_per_file = groups_per_file//tiles_groups_per_chunk
permut = np.random.permutation(chunks_per_file)
ds['corr2d'] = ds['corr2d']. reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
ds['target_disparity'] = ds['target_disparity'].reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
ds['gt_ds'] = ds['gt_ds']. reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
def replaceDataset(datasets_data, new_dataset, nset, period, findx):
"""
Replace one file in the dataset
"""
datasets_data[findx]['corr2d'] [nset::period] = new_dataset['corr2d']
datasets_data[findx]['target_disparity'][nset::period] = new_dataset['target_disparity']
datasets_data[findx]['gt_ds'] [nset::period] = new_dataset['gt_ds']
def zip_lvar_hvar(datasets_all_data, del_src = True):
# cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
# Reformat input data
num_sets_to_combine = len(datasets_all_data)
datasets_data = []
if num_sets_to_combine:
for nrec in range(len(datasets_all_data[0])):
recs = [[] for _ in range(num_sets_to_combine)]
for nset, datasets in enumerate(datasets_all_data):
recs[nset] = datasets[nrec]
rec = {'corr2d': np.empty((recs[0]['corr2d'].shape[0]*num_sets_to_combine, recs[0]['corr2d'].shape[1]),dtype=np.float32),
'target_disparity': np.empty((recs[0]['target_disparity'].shape[0]*num_sets_to_combine,recs[0]['target_disparity'].shape[1]),dtype=np.float32),
'gt_ds': np.empty((recs[0]['gt_ds'].shape[0]*num_sets_to_combine, recs[0]['gt_ds'].shape[1]),dtype=np.float32)}
for nset, reci in enumerate(recs):
rec['corr2d'] [nset::num_sets_to_combine] = recs[nset]['corr2d']
rec['target_disparity'][nset::num_sets_to_combine] = recs[nset]['target_disparity']
rec['gt_ds'] [nset::num_sets_to_combine] = recs[nset]['gt_ds']
if del_src:
for nset in range(num_sets_to_combine):
datasets_all_data[nset][nrec] = None
datasets_data.append(rec)
return datasets_data
# list of dictionaries
def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
if (not datasets_data is None) and (len (datasets_data) > 0):
tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
tss = int(np.sqrt(tsz)+0.5)
offs = (tss - reduced_tile_side) // 2
for rec in datasets_data:
rec['corr2d'] = (rec['corr2d'].reshape((-1, num_tile_layers, tss, tss))
[..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0,
radius = 0):
# for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(result_file)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool)
not_nan_ext[radius:-radius,radius:-radius] = not_nan
for dy in range(2*radius+1):
for dx in range(2*radius+1):
not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
not_nan = not_nan_ext[radius:-radius,radius:-radius]
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
rslt.append([rms0,rms1])
return rslt
"""
Start of the main code
"""
"""
try:
train_filenameTFR = sys.argv[1]
except IndexError:
train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
try:
test_filenameTFR = sys.argv[2]
except IndexError:
test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
#FILES_PER_SCENE
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
"""
data_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
data_dir1 = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
img_dir = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
dir_train_lvar = data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar = data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_lvar1 = data_dir1 #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_train_hvar1 = data_dir1 # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_lvar = data_dir # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_test_hvar = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_3" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
dir_img = os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
dir_result = os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
files_train_lvar1 = ["train000_R2_LE_1.5.tfrecords",
"train001_R2_LE_1.5.tfrecords",
"train002_R2_LE_1.5.tfrecords",
"train003_R2_LE_1.5.tfrecords",
"train004_R2_LE_1.5.tfrecords",
"train005_R2_LE_1.5.tfrecords",
"train006_R2_LE_1.5.tfrecords",
"train007_R2_LE_1.5.tfrecords",
]
files_train_hvar1 = ["train000_R2_GT_1.5.tfrecords",
"train001_R2_GT_1.5.tfrecords",
"train002_R2_GT_1.5.tfrecords",
"train003_R2_GT_1.5.tfrecords",
"train004_R2_GT_1.5.tfrecords",
"train005_R2_GT_1.5.tfrecords",
"train006_R2_GT_1.5.tfrecords",
"train007_R2_GT_1.5.tfrecords",
]
"""
Try again - all hvar training, train than different hvar testing.
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
match for either of hvar/lvar, even those that were used for training
tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
try same with higher LR - will they eventually converge?
Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
"""
#just testing:
#files_train_lvar = files_train_hvar
files_test_lvar = ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
files_test_hvar = ["train000_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
#files_img = ['1527257933_150165-v04'] # overlook
#files_img = ['1527256858_150165-v01'] # state Street
files_img = ['1527256816_150165-v02'] # staete Street
for i, path in enumerate(files_train_lvar):
files_train_lvar[i]=os.path.join(dir_train_lvar, path)
for i, path in enumerate(files_train_hvar):
files_train_hvar[i]=os.path.join(dir_train_hvar, path)
# Second set of files
for i, path in enumerate(files_train_lvar1):
files_train_lvar1[i]=os.path.join(dir_train_lvar1, path)
for i, path in enumerate(files_train_hvar1):
files_train_hvar1[i]=os.path.join(dir_train_hvar1, path)
for i, path in enumerate(files_test_lvar):
files_test_lvar[i]=os.path.join(dir_test_lvar, path)
for i, path in enumerate(files_test_hvar):
files_test_hvar[i]=os.path.join(dir_test_hvar, path)
result_files=[]
for i, path in enumerate(files_img):
files_img[i] = os.path.join(dir_img, path+'.tfrecords')
result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
#file_test_hvar= None
weight_hvar = 0.26
weight_lvar = 1.0 - weight_hvar
import tensorflow as tf
import tensorflow.contrib.slim as slim
for result_file in result_files:
try:
eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
#exit(0)
except:
pass
datasets_img = []
for fpath in files_img:
print_time("Importing test image data from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_img.append( {"corr2d": corr2d,
"target_disparity": target_disparity,
"gt_ds": gt_ds})
print_time(" Done")
gtruth = datasets_img[0]['gt_ds'].copy()
t_disp = datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
#reformat_to_clusters(datasets_img) already this format
replace_nan(datasets_img)
pass
pass
datasets_train_lvar = []
datasets_train_hvar = []
datasets_train_lvar1 = []
datasets_train_hvar1 = []
for fpath in files_train_lvar:
print_time("Importing train data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
for fpath in files_train_hvar:
print_time("Importing train data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
if TWO_TRAINS:
for fpath in files_train_lvar1:
print_time("Importing train data1 (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_lvar1.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
for fpath in files_train_hvar1:
print_time("Importing train data1 (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_train_hvar1.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_lvar = []
for fpath in files_test_lvar:
print_time("Importing test data (low variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_lvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
datasets_test_hvar = []
for fpath in files_test_hvar:
print_time("Importing test data (high variance) from "+fpath, end="")
corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
datasets_test_hvar.append({"corr2d":corr2d,
"target_disparity":target_disparity,
"gt_ds":gt_ds})
print_time(" Done")
#CLUSTER_RADIUS
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
# Reformat input data
if FILE_TILE_SIDE > TILE_SIDE:
print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_lvar1, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_train_hvar1, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_lvar, TILE_LAYERS, TILE_SIDE)
reduce_tile_size(datasets_test_hvar, TILE_LAYERS, TILE_SIDE)
print_time(" Done")
pass
print_time("Reshaping train data (low variance)", end="")
reformat_to_clusters(datasets_train_lvar)
print_time(" Done")
print_time("Reshaping train data (high variance)", end="")
reformat_to_clusters(datasets_train_hvar)
print_time(" Done")
print_time("Reshaping train data1 (low variance)", end="")
reformat_to_clusters(datasets_train_lvar1)
print_time(" Done")
print_time("Reshaping train data1 (high variance)", end="")
reformat_to_clusters(datasets_train_hvar1)
print_time(" Done")
print_time("Reshaping test data (low variance)", end="")
reformat_to_clusters(datasets_test_lvar)
print_time(" Done")
print_time("Reshaping test data (high variance)", end="")
reformat_to_clusters(datasets_test_hvar)
print_time(" Done")
pass
"""
datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
test has to have even original, batches will not zip - just use two batches for one big one
"""
datasets_train_list = [datasets_train_lvar, datasets_train_hvar]
if TWO_TRAINS:
datasets_train_list += [datasets_train_lvar1, datasets_train_hvar1]
if ZIP_LHVAR:
print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
datasets_train = zip_lvar_hvar(datasets_train_list, del_src = True) # no shuffle, delete src
print_time(" Done")
else:
#Alternate lvar/hvar
datasets_train = []
datasets_weights_train = []
for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
if (indx < len(datasets_train_lvar)):
datasets_train.append(datasets_train_lvar[indx])
datasets_weights_train.append(weight_lvar)
if (indx < len(datasets_train_hvar)):
datasets_train.append(datasets_train_hvar[indx])
datasets_weights_train.append(weight_hvar)
datasets_test = []
datasets_weights_test = []
for dataset_test_lvar in datasets_test_lvar:
datasets_test.append(dataset_test_lvar)
datasets_weights_test.append(weight_lvar)
for dataset_test_hvar in datasets_test_hvar:
datasets_test.append(dataset_test_hvar)
datasets_weights_test.append(weight_hvar)
corr2d_train_placeholder = tf.placeholder(datasets_train[0]['corr2d'].dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.placeholder(datasets_train[0]['gt_ds'].dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
#dataset_tt TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d":corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
#dataset_train_size = len(corr2d_train)
#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
dataset_train_size = len(datasets_train[0]['corr2d'])
dataset_train_size //= BATCH_SIZE
dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(datasets_img[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
"""
next_element_tt dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}
'corr2d' (140405473715624) Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)
'gt_ds' (140405473715680) Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)
'target_disparity' (140405501995888) Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)
"""
#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * TILE_SIDE * TILE_SIDE
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input, layout, reuse, sym8 = False):
last_indx = None;
fc = []
inp_weights = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input
if sym8:
inp8 = sym_inputs8(inp)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
reuse_this = reuse | (j > 0)
scp = 'g_fc_sub'+str(i)
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope= scp, reuse = reuse_this))
if not reuse_this:
with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
if num_non_sum > 0:
reuse_this = reuse
scp = 'g_fc_sub'+str(i)+"r"
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope=scp, reuse = reuse_this))
if not reuse_this:
with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
scp = 'g_fc_sub'+str(i)
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope= scp, reuse = reuse))
if not reuse:
with tf.variable_scope(scp, reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
return fc[-1], inp_weights
def network_inter(input, layout):
last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
inp = fc[-1]
else:
inp = input
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
if USE_CONFIDENCE:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu,scope='g_fc_inter_out')
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None,scope='g_fc_inter_out')
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def network_siam(input, # now [?,9,325]-> [?,25,325]
layout1,
layout2,
sym8 = False,
only_tile = None): # just for debugging - feed only data from the center sub-network
with tf.name_scope("Siam_net"):
inp_weights = []
num_legs = input.shape[1] # == 9
inter_list = []
reuse = False
for i in range (num_legs):
if (only_tile is None) or (i == only_tile):
# inter_list.append(network_sub(input[:,i,:],
# layout= layout1,
# reuse= reuse,
# sym8 = sym8))
ns, ns_weights = network_sub(input[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8)
inter_list.append(ns)
inp_weights += ns_weights
reuse = True
inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
return network_inter (inter_tensor, layout2), inp_weights
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
tf_num_tiles = tf.shape(gt_ds_batch)[0]
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
absolute_disparity = True, #when false there should be no activation on disparity output !
use_confidence = True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
# w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1], name = "w_gt_slice")
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
# w_clip = tf.clip_by_value(w_sub, tf_0f,tf_maxw, name = "w_clip")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
# conf_slice = tf.slice(out_batch,[0,1],[-1,1], name = "conf_slice")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
out_wdiff2 = tf.multiply (out_diff2, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
# dispw_comp = tf.multiply (w_norm, w_norm, name = "dispw_comp") #HERE??
dispw_sum = tf.reduce_sum(dispw_comp, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_comp, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
def weightsLoss(inp_weights): # [batch_size,(1..2)] tf_result
# weights_lambdas): # single lambda or same length as inp_weights.shape[1]
"""
Enforcing 'smooth' weights for the input 2d correlation tiles
@return mean squared difference for each weight and average of 8 neighbors divided by mean squared weights
"""
weight_ortho = 1.0
weight_diag = 0.7
sw = 4.0 * (weight_ortho + weight_diag)
weight_ortho /= sw
weight_diag /= sw
# w_neib = tf.const([[weight_diag, weight_ortho, weight_diag],
# [weight_ortho, -1.0, weight_ortho],
# [weight_diag, weight_ortho, weight_diag]])
with tf.name_scope("WeightsLoss"):
# Adding 1 tile border
tf_inp = tf.reshape(inp_weights[:TILE_LAYERS * TILE_SIZE,:], [TILE_LAYERS, FILE_TILE_SIDE, FILE_TILE_SIDE, inp_weights.shape[1]], name = "tf_inp")
tf_inp_ext_h = tf.concat([tf_inp [:, :, :1, :], tf_inp, tf_inp [:, :, -1:, :]], axis = 2, name ="tf_inp_ext_h")
tf_inp_ext = tf.concat([tf_inp_ext_h [:, :1, :, :], tf_inp_ext_h, tf_inp_ext_h[:, -1:, :, :]], axis = 1, name ="tf_inp_ext")
s_ortho = tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:,:] + tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:, :]
s_corn = tf_inp_ext[:, :-2,:-2,:] + tf_inp_ext[:, :-2, 2:,:] + tf_inp_ext[:,2:, :-2,:] + tf_inp_ext[:,2: , 2:, :]
w_diff = tf.subtract(tf_inp, s_ortho * weight_ortho + s_corn * weight_diag, name="w_diff")
w_diff2 = tf.multiply(w_diff, w_diff, name="w_diff2")
w_var = tf.reduce_mean(w_diff2, name="w_var")
w2_mean = tf.reduce_mean(inp_weights * inp_weights, name="w2_mean")
w_rel = tf.divide(w_var, w2_mean, name= "w_rel")
return w_rel # scalar, cost for weights non-smoothness in 2d
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
out, inp_weights = network_siam(input=corr2d_Nx325,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE) #Remove/put None for normal operation
# w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1], name = "target_center")
#gt_ds_batch_center = tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1], [-1,2], name = "gt_ds_center")
G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch = out, # [batch_size,(1..2)] tf_result
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
if WLOSS_LAMBDA > 0.0:
W_loss = weightsLoss(inp_weights[0]) # inp_weights - list of tensors, currently - just [0]
GW_loss = tf.add(G_loss, WLOSS_LAMBDA * W_loss, name = "GW_loss")
else:
GW_loss = G_loss
W_loss = 0.0
#debug
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_W_loss = tf.placeholder(tf.float32,shape=None,name='W_loss_avg')
tf_ph_GW_loss = tf.placeholder(tf.float32,shape=None,name='GW_loss_avg')
tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
with tf.name_scope('sample'):
tf.summary.scalar("GW_loss", GW_loss)
tf.summary.scalar("G_loss", G_loss)
tf.summary.scalar("W_loss", W_loss)
tf.summary.scalar("sq_diff", _cost1)
tf.summary.scalar("gtvar_diff", GT_variance)
with tf.name_scope('epoch_average'):
tf.summary.scalar("GW_loss_epoch", tf_ph_GW_loss)
tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.summary.scalar("W_loss_epoch", tf_ph_W_loss)
tf.summary.scalar("sq_diff_epoch", tf_ph_sq_diff)
tf.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.summary.scalar("img_test0", tf_img_test0)
tf.summary.scalar("img_test9", tf_img_test9)
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(GW_loss)
saver=tf.train.Saver()
ROOT_PATH = './attic/nn_ds_neibs8_graph'+SUFFIX+"/"
TRAIN_PATH = ROOT_PATH + 'train'
TEST_PATH = ROOT_PATH + 'test'
TEST_PATH1 = ROOT_PATH + 'test1'
# CLEAN OLD STAFF
shutil.rmtree(TRAIN_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH, ignore_errors=True)
shutil.rmtree(TEST_PATH1, ignore_errors=True)
WIDTH=324
HEIGHT=242
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(TRAIN_PATH, sess.graph)
test_writer = tf.summary.FileWriter(TEST_PATH, sess.graph)
test_writer1 = tf.summary.FileWriter(TEST_PATH1, sess.graph)
loss_gw_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_g_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_w_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_gw_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_g_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_w_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
train_gw_avg = 0.0
train_g_avg = 0.0
train_w_avg = 0.0
test_gw_avg = 0.0
test_g_avg = 0.0
test_w_avg = 0.0
train2_avg = 0.0
test2_avg = 0.0
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
gtvar_train_avg = 0.0
gtvar_test_avg = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
num_train_variants = len(datasets_train)
for epoch in range (EPOCHS_TO_RUN):
# file_index = (epoch // 20) % 2
file_index = epoch % num_train_variants
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
# print ("sr1",file=sys.stderr,end=" ")
if (file_index == 0) and SHUFFLE_FILES:
num_sets = len(datasets_train_list)
print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
for i in range(num_sets):
shuffle_in_place (datasets_train, i, num_sets)
print_time(" Done")
print_time("Shuffling tile chunks ", end="")
shuffle_chunks_in_place (datasets_train, 1)
print_time(" Done")
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: datasets_train[file_index]['corr2d'],
target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
gt_ds_train_placeholder: datasets_train[file_index]['gt_ds']})
for i in range(dataset_train_size):
try:
train_summary,_, GW_loss_trained, G_loss_trained, W_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ merged,
G_opt,
GW_loss,
G_loss,
W_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr: learning_rate,
tf_ph_GW_loss: train_gw_avg,
tf_ph_G_loss: train_g_avg,
tf_ph_W_loss: train_w_avg,
tf_ph_sq_diff: train2_avg,
tf_gtvar_diff: gtvar_train_avg,
tf_img_test0: img_gain_test0,
tf_img_test9:img_gain_test9}) # previous value of *_avg
loss_gw_train_hist[i] = GW_loss_trained
loss_g_train_hist[i] = G_loss_trained
loss_w_train_hist[i] = W_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("train done at step %d"%(i))
break
train_gw_avg = np.average(loss_gw_train_hist).astype(np.float32)
train_g_avg = np.average(loss_g_train_hist).astype(np.float32)
train_w_avg = np.average(loss_w_train_hist).astype(np.float32)
train2_avg = np.average(loss2_train_hist).astype(np.float32)
gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
test_summaries = [0.0]*len(datasets_test)
tst_avg = [0.0]*len(datasets_test)
tst2_avg = [0.0]*len(datasets_test)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test['corr2d'],
target_disparity_train_placeholder: dataset_test['target_disparity'],
gt_ds_train_placeholder: dataset_test['gt_ds']})
for i in range(dataset_test_size):
try:
test_summaries[ntest], GW_loss_tested, G_loss_tested, W_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
GW_loss,
G_loss,
W_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr: learning_rate,
tf_ph_GW_loss: test_gw_avg,
tf_ph_G_loss: test_g_avg,
tf_ph_W_loss: test_w_avg,
tf_ph_sq_diff: test2_avg,
tf_gtvar_diff: gtvar_test_avg,
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9}) # previous value of *_avg
loss_gw_test_hist[i] = GW_loss_tested
loss_g_test_hist[i] = G_loss_tested
loss_w_test_hist[i] = W_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
test_gw_avg = np.average(loss_gw_test_hist).astype(np.float32)
test_g_avg = np.average(loss_g_test_hist).astype(np.float32)
test_w_avg = np.average(loss_w_test_hist).astype(np.float32)
tst_avg[ntest] = test_gw_avg
test2_avg = np.average(loss2_test_hist).astype(np.float32)
tst2_avg[ntest] = test2_avg
gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
train_writer.add_summary(train_summary, epoch)
test_writer.add_summary(test_summaries[0], epoch)
test_writer1.add_summary(test_summaries[1], epoch)
print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_gw_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
###################################################
# Read the full image
###################################################
test_summaries_img = [0.0]*len(datasets_img)
# disp_out= np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest,dataset_img in enumerate(datasets_img):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
try:
test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[merged,
G_loss,
out,
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={lr: learning_rate,
tf_ph_GW_loss: test_gw_avg,
tf_ph_G_loss: test_g_avg,
tf_ph_W_loss: test_w_avg,
tf_ph_sq_diff: test2_avg,
tf_gtvar_diff: gtvar_test_avg,
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9}) # previous value of *_avg
except tf.errors.OutOfRangeError:
print("test done at step %d"%(i))
break
try:
disp_out[start_offs:end_offs] = output.flatten()
except ValueError:
print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
break;
pass
result_file = result_files[ntest]
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
# Close writers
train_writer.close()
test_writer.close()
test_writer1.close()
#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
print("All done")
exit (0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment