Commit b1111bca authored by Andrey Filippov's avatar Andrey Filippov

Trying versions and working on new evaluation criteria for FG/BG

features
parent 2c44046d
#!/usr/bin/env python3
#from numpy import float64
from tensorflow.contrib.image.ops.gen_distort_image_ops import adjust_hsv_in_yiq
#from tensorflow.contrib.image.ops.gen_distort_image_ops import adjust_hsv_in_yiq
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
......@@ -161,7 +161,7 @@ def writeTFRecordsFromImageSet(
s_gt, # wght, # will be modified in place
w_diag = 0.7,
w_reduce = 0.7,
num_pass = 10,
num_pass = 50,
eps = 1E-6)
if debug > 1:
......@@ -281,7 +281,8 @@ def fillGapsByLaplacian(
w_diag = 0.7,
w_reduce = 0.7,
num_pass = 10,
eps = 1E-6):
eps = 1E-6,
debug_level = 0):
dirs = ((-1,0), (-1,1), (0,1), (1,1), (1,0), (1,-1), (0,-1), (-1,-1))
wneib = ( 1.0, w_diag, 1.0, w_diag, 1.0, w_diag, 1.0, w_diag)
gap_tiles = []
......@@ -304,7 +305,7 @@ def fillGapsByLaplacian(
valn = np.copy(val)
wghtn = np.copy(wght)
achange = eps * np.max(wght)
for _ in range (num_pass):
for npass in range (num_pass):
num_new = 1
max_diff = 0.0;
for tile, neibs in zip (gap_tiles, gap_neibs):
......@@ -326,6 +327,8 @@ def fillGapsByLaplacian(
max_diff = max(max_diff, wdiff)
np.copyto(val, valn)
np.copyto(wght, wghtn)
if (debug_level > 3):
print("Pass %d, max_diff = %f"%(npass, max_diff))
if (num_new == 0) and (max_diff < achange):
break
......@@ -536,13 +539,23 @@ class ExploreData:
ds_pair[:,:,:,2] = gtaux[:,:,:, ijt.IJFGBG.AUX_DISP] # 8
for nf in range (ds_pair.shape[0]):
if (self.debug_level > 3):
print ("---- nf=%d"%(nf,))
fillGapsByLaplacian(
ds_pair[nf,:,:,0], # val, # will be modified in place
ds_pair[nf,:,:,1], # wght, # will be modified in place
w_diag = 0.7,
w_reduce = 0.7,
num_pass = 20,
eps = 1E-6)
eps = 1E-6,
debug_level = self.debug_level)
if (self.debug_level > 0):
print ("---- nf=%d min = %f mean = %f max = %f"%(
nf,
ds_pair[nf,:,:,0].min(),
ds_pair[nf,:,:,0].mean(),
ds_pair[nf,:,:,0].max()))
print("zero strength",np.nonzero(ds_pair[nf,:,:,1]==0.0))
return ds_pair
......@@ -847,6 +860,10 @@ class ExploreData:
min_neibs = None, # Minimal number of valid tiles to include
use_split = False, # Select y single/multi-plane tiles (center only)
keep_split = False):# When sel_split, keep only multi-plane tiles (false - only single-plane)
#for file names:
self.min_neibs = min_neibs
self.use_split = use_split
self.keep_split = keep_split
if data_ds is None:
data_ds = self.train_ds
......@@ -903,6 +920,7 @@ class ExploreData:
'''
def augmentBatchFileIndices(self,
seed_index,
seed_list = None,
min_choices=None,
max_files = None,
set_ds = None
......@@ -915,14 +933,21 @@ class ExploreData:
set_ds = self.train_ds
full_num_choices = self.num_batch_tiles[seed_index].copy()
flist = [seed_index]
all_choices = list(range(self.num_batch_tiles.shape[0]))
all_choices.remove(seed_index)
if seed_list is None:
seed_list = list(range(self.num_batch_tiles.shape[0]))
all_choices = list(seed_list) # a copy of seed list
all_choices.remove(seed_index) # seed_list made unique by the caller
### list(filter(lambda a: a != seed_index, all_choices)) # remove all instances of seed_index
for _ in range (max_files-1):
if full_num_choices.min() >= min_choices:
break
if len(all_choices) == 0:
print ("Nothing left in all_choices!")
break
findx = np.random.choice(all_choices)
flist.append(findx)
all_choices.remove(findx)
all_choices.remove(findx) # seed_list made unique by the caller
### list(filter(lambda a: a != findx, all_choices)) # remove all instances of findx
full_num_choices += self.num_batch_tiles[findx]
file_tiles_sparse = [[] for _ in set_ds] #list of empty lists for each train scene (will be sparse)
......@@ -1012,6 +1037,7 @@ ValueError: need at least one array to concatenate
def prepareBatchData(self,
ml_list,
seed_index,
seed_list,
min_choices=None,
max_files = None,
ml_num = None,
......@@ -1036,7 +1062,12 @@ ValueError: need at least one array to concatenate
width_m1 = width-1
height_m1 = height-1
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
flist,tiles = self.augmentBatchFileIndices(
seed_index,
seed_list,
min_choices,
max_files,
set_ds)
ml_all_files = self.getBatchData(
flist,
ml_list,
......@@ -1137,6 +1168,7 @@ ValueError: need at least one array to concatenate
writer = tf.io.TFRecordWriter(tfr_filename)
if num_scenes is None:
num_scenes = len(files_list)
'''
if len(files_list) <= num_scenes:
#create and shuffle repetitive list of files of num_scenes.length
seed_list = np.arange(num_scenes) % len(files_list)
......@@ -1147,11 +1179,25 @@ ValueError: need at least one array to concatenate
np.random.shuffle(seed_list)
seed_list = seed_list[:num_scenes]
np.random.shuffle(seed_list)
'''
augment_list = []
for seed_indx in np.arange(len(files_list)):
if self.num_batch_tiles[seed_indx].sum() >0:
augment_list.append(seed_indx)
seed_list = list(augment_list) # seed list will be modified while augment_list will have unique/full list of suitable files
while len(seed_list) < num_scenes:
seed_list.append(np.random.choice(seed_list))
np.random.shuffle(seed_list)
if len(seed_list) >= num_scenes:
seed_list = seed_list[:num_scenes]
cluster_size = (2 * radius + 1) * (2 * radius + 1)
for nscene, seed_index in enumerate(seed_list):
corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData( #'hor-pairs' is not in list
ml_list,
seed_index,
augment_list,
min_choices=None,
max_files = None,
ml_num = None,
......@@ -1195,6 +1241,7 @@ ValueError: need at least one array to concatenate
sweep_files,
sweep_disparities,
seed_index,
seed_list,
min_choices=None,
max_files = None,
set_ds = None,
......@@ -1210,8 +1257,6 @@ ValueError: need at least one array to concatenate
min_choices = self.min_batch_choices #10
if max_files is None:
max_files = self.max_batch_files #10
### if ml_num is None:
### ml_num = self.files_per_scene #5 ????
if set_ds is None:
set_ds = self.train_ds
tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
......@@ -1223,6 +1268,7 @@ ValueError: need at least one array to concatenate
flist0, tiles0 = self.augmentBatchFileIndices(
seed_index,
seed_list,
min_choices,
max_files,
set_ds)
......@@ -1306,6 +1352,13 @@ ValueError: need at least one array to concatenate
rnd_plate = 0.0):## disparity random for each plate (now 25 tiles)
# open the TFRecords file
fb = ""
if self.use_split:
fb = ["-FB1","-FB2"][self.keep_split] # single plane - FB1, split FG/BG planes - FB2
tfr_filename+="-RT%1.2f-RP%1.2f-M%d-NB%d%s"%(rnd_tile,rnd_plate,self.fgbg_mode,self.min_neibs, fb)
if not '.tfrecords' in tfr_filename:
tfr_filename += '.tfrecords'
......@@ -1324,10 +1377,11 @@ ValueError: need at least one array to concatenate
#skip writing if file exists - it will be possible to continue or run several instances
if os.path.exists(tfr_filename):
print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
# return # Temporary disable
return # Temporary disable
writer = tf.io.TFRecordWriter(tfr_filename)
if num_scenes is None:
num_scenes = len(files_list)
'''
if len(files_list) <= num_scenes:
#create and shuffle repetitive list of files of num_scenes.length
seed_list = np.arange(num_scenes) % len(files_list)
......@@ -1337,7 +1391,19 @@ ValueError: need at least one array to concatenate
seed_list = np.arange(len(files_list))
np.random.shuffle(seed_list)
seed_list = seed_list[:num_scenes]
'''
augment_list = []
for seed_indx in np.arange(len(files_list)):
if self.num_batch_tiles[seed_indx].sum() >0:
augment_list.append(seed_indx)
seed_list = list(augment_list) # seed list will be modified while augment_list will have unique/full list of suitable files
while len(seed_list) < num_scenes:
seed_list.append(np.random.choice(seed_list))
np.random.shuffle(seed_list)
if len(seed_list) >= num_scenes:
seed_list = seed_list[:num_scenes]
cluster_size = (2 * radius + 1) * (2 * radius + 1)
for nscene, seed_index in enumerate(seed_list):
corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchDataLwir( #'hor-pairs' is not in list
......@@ -1345,9 +1411,9 @@ ValueError: need at least one array to concatenate
sweep_files = sweep_files,
sweep_disparities = sweep_disparities,
seed_index = seed_index,
seed_list = augment_list,
min_choices = None,
max_files = None,
### ml_num = None,
set_ds = set_ds, #DS data from all GT_AX files scanned
radius = radius,
rnd_tile = rnd_tile, ## disparity random for each tile
......@@ -1463,7 +1529,7 @@ if __name__ == "__main__":
## topdir_train = "/data_ssd/data_sets/train_mlr32_18d"
## topdir_train = '/data_ssd/data_sets/test_only'# ''
### topdir_train = '/data_ssd/data_sets/train_set2'# ''
topdir_train = '/data_ssd/lwir_sets/lwir_train1'# ''
topdir_train = '/data_ssd/lwir_sets/lwir_train2'# ''
# tf_data_5x5_main_10_heur
try:
......@@ -1472,8 +1538,8 @@ if __name__ == "__main__":
# topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
# topdir_test = "/data_ssd/data_sets/test_mlr32_18d"
## topdir_test = '/data_ssd/data_sets/test_only'
### topdir_test = '/data_ssd/data_sets/test_set2'
topdir_test = '/data_ssd/lwir_sets/lwir_test1'
### topdir_test = '/data_ssd/data_sets/test_set21'
topdir_test = '/data_ssd/lwir_sets/lwir_test2'
try:
......@@ -1482,7 +1548,7 @@ if __name__ == "__main__":
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
### pathTFR = "/data_ssd/data_sets/tf_data_5x5_main_13_heur"
pathTFR = '/data_ssd/lwir_sets/tf_data_5x5_01'
pathTFR = '/data_ssd/lwir_sets/tf_data_5x5_2'
## pathTFR = "/data_ssd/data_sets/tf_data_5x5_main_11_rnd"
## pathTFR = "/data_ssd/data_sets/tf_data_5x5_main_12_rigrnd"
......@@ -1589,7 +1655,7 @@ if __name__ == "__main__":
FRAC_NEIBS_VALID = 0.55# 8 #LWIR new
MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
MIN_NEIBS = round (MIN_NEIBS * FRAC_NEIBS_VALID)
VARIANCE_THRESHOLD = 0.4 # 1.5
VARIANCE_THRESHOLD = 0.8 # 0.4 # 1.5
VARIANCE_SCALE_DISPARITY = 5.0 #Scale variance if average is above this
NUM_TRAIN_SETS = 32 # 8
......@@ -1600,6 +1666,9 @@ if __name__ == "__main__":
RND_AMPLIUDE_TRAIN_PLATE = 0.5 # train with corr2d rendered +/- this far from the GT - common for each (5x5) plate component
MAX_MAIN_OFFSET = 2.5 # do not use tile for training if MAIN camera (AUX for LWIR) differs more from GT
MODEL_ML_DIR = "ml32" # subdirectory with the ML disparity sweep files
USE_SPLIT = False # True, # Select y single/multi-plane tiles (center only)
KEEP_SPLIT = False # When sel_split, keep only multi-plane tiles (false - only single-plane)
if not topdir_train:
NUM_TRAIN_SETS = 0
......@@ -1618,14 +1687,14 @@ if __name__ == "__main__":
''' Prepare full image for testing '''
"""
for model_ml_path in test_sets:
writeTFRecordsFromImageSet(
model_ml_path, # model/version/ml_dir
FGBGMODE_TEST, # 0, # expot_mode, # 0 - GT average, 1 - GT FG, 2 - GT BG, 3 - AUX disparity
RND_AMPLIUDE_TEST, # random_offset, # for modes 0..2 - add random offset of -random_offset to +random_offset, in mode 3 add random to GT average if no AUX data
pathTFR) # TFR directory
"""
# disp_bins = 20,
......@@ -1641,13 +1710,13 @@ if __name__ == "__main__":
ml_pattern = ml_pattern,
max_main_offset = MAX_MAIN_OFFSET,
latest_version_only = LATEST_VERSION_ONLY,
debug_level = 3, #1, #3, ##0, #3,
debug_level = 1, #3, #1, #3, ##0, #3,
disparity_bins = 50, #100 #200, #1000,
strength_bins = 50, #100
disparity_min_drop = -0.1,
disparity_min_clip = -0.1,
disparity_max_drop = 5.0, #100.0,
disparity_max_clip = 5.0, #100.0,
disparity_max_drop = 8.0, #100.0,
disparity_max_clip = 8.0, #100.0,
strength_min_drop = 0.02, # 0.1,
strength_min_clip = 0.02, # 0.1,
strength_max_drop = 0.3, # 1.0,
......@@ -1743,13 +1812,21 @@ if __name__ == "__main__":
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
scale_disp = VARIANCE_SCALE_DISPARITY,
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
pass
for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds)
ex_data.writeTFRewcordsEpochLwir(
fpath,
sweep_files = ex_data.train_sweep_files,
sweep_disparities = ex_data.train_sweep_disparities,
files_list = ex_data.files_train,
set_ds = ex_data.train_ds,
radius = ex_data.radius,
rnd_tile = ex_data.rnd_tile,
rnd_plate = ex_data.rnd_plate)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds,
......@@ -1759,12 +1836,21 @@ if __name__ == "__main__":
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
fpath = test_filenameTFR # +("-%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, num_scenes = num_test_scenes)
pass
ex_data.writeTFRewcordsEpochLwir(
fpath,
sweep_files = ex_data.test_sweep_files,
sweep_disparities = ex_data.test_sweep_disparities,
files_list = ex_data.files_test,
set_ds = ex_data.test_ds,
radius = ex_data.radius,
num_scenes = num_test_scenes,
rnd_tile = ex_data.rnd_tile,
rnd_plate = ex_data.rnd_plate)
else: # RADIUS > 0
# test
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
......@@ -1775,8 +1861,8 @@ if __name__ == "__main__":
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
num_le_test = num_batch_tiles_test.sum()
print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
......@@ -1802,8 +1888,8 @@ if __name__ == "__main__":
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
num_gt_test = num_batch_tiles_test.sum()
high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
......@@ -1831,8 +1917,8 @@ if __name__ == "__main__":
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
num_le_fake = num_batch_tiles_fake.sum()
print("Number of <= %f disparity variance tiles: %d (test)"%(VARIANCE_THRESHOLD, num_le_fake))
......@@ -1858,8 +1944,8 @@ if __name__ == "__main__":
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
num_gt_fake = num_batch_tiles_fake.sum()
high_fract_fake = 1.0 * num_gt_fake / (num_le_fake + num_gt_fake)
......@@ -1888,8 +1974,8 @@ if __name__ == "__main__":
min_var = 0.0, # Minimal tile variance to include
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
num_le_train = num_batch_tiles_train.sum()
print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train))
......@@ -1914,8 +2000,8 @@ if __name__ == "__main__":
min_var = VARIANCE_THRESHOLD, # Minimal tile variance to include
max_var = 1000.0, # Maximal tile variance to include
min_neibs = MIN_NEIBS, # Minimal number of valid tiles to include
use_split = True, # Select y single/multi-plane tiles (center only)
keep_split = False) # When sel_split, keep only multi-plane tiles (false - only single-plane)
use_split = USE_SPLIT, # Select y single/multi-plane tiles (center only)
keep_split = KEEP_SPLIT) # When sel_split, keep only multi-plane tiles (false - only single-plane)
num_gt_train = num_batch_tiles_train.sum()
high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train)
......
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
#!/usr/bin/env python3
'''
/**
* @file imagej_tiffwriter.py
* @brief save tiffs for imagej (1.52d+) - with stacks and hyperstacks
* @par <b>License</b>:
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
'''
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "oleg@elphel.com"
'''
Usage example:
import imagej_tiffwriter
import numpy as np
Have a few images in the form of numpy arrays np.float32:
- (h,w)
- (n,h,w)
Labels can be provided as a list: ['label1','label2', etc.]
No list length check against number of images
imagej_tiffwriter.save(path,images,labels)
'''
import numpy as np
import struct
import tifffile
import math
# from here: https://stackoverflow.com/questions/50258287/how-to-specify-colormap-when-saving-tiff-stack
def imagej_metadata_tags(metadata, byteorder):
"""Return IJMetadata and IJMetadataByteCounts tags from metadata dict.
The tags can be passed to the TiffWriter.save function as extratags.
"""
header = [{'>': b'IJIJ', '<': b'JIJI'}[byteorder]]
bytecounts = [0]
body = []
def writestring(data, byteorder):
return data.encode('utf-16' + {'>': 'be', '<': 'le'}[byteorder])
def writedoubles(data, byteorder):
return struct.pack(byteorder+('d' * len(data)), *data)
def writebytes(data, byteorder):
return data.tobytes()
metadata_types = (
('Info', b'info', 1, writestring),
('Labels', b'labl', None, writestring),
('Ranges', b'rang', 1, writedoubles),
('LUTs', b'luts', None, writebytes),
('Plot', b'plot', 1, writebytes),
('ROI', b'roi ', 1, writebytes),
('Overlays', b'over', None, writebytes))
for key, mtype, count, func in metadata_types:
if key not in metadata:
continue
if byteorder == '<':
mtype = mtype[::-1]
values = metadata[key]
if count is None:
count = len(values)
else:
values = [values]
header.append(mtype + struct.pack(byteorder+'I', count))
for value in values:
data = func(value, byteorder)
body.append(data)
bytecounts.append(len(data))
body = b''.join(body)
header = b''.join(header)
data = header + body
bytecounts[0] = len(header)
bytecounts = struct.pack(byteorder+('I' * len(bytecounts)), *bytecounts)
return ((50839, 'B', len(data), data, True),
(50838, 'I', len(bytecounts)//4, bytecounts, True))
#def save(path,images,force_stack=False,force_hyperstack=False):
def save(path,images,labels=None,label_prefix="Label "):
'''
labels a list or None
'''
'''
Expecting:
(h,w),
(n,h,w) - just create a simple stack
'''
# Got images, analyze shape:
# - possible formats (c == depth):
# -- (t,z,h,w,c)
# -- (t,h,w,c), t or z does not matter
# -- (h,w,c)
# -- (h,w)
# 0 or 1 images.shapes are not handled
#
# (h,w)
if len(images.shape)==2:
images = images[np.newaxis,...]
# now the shape length is 3
if len(images.shape)==3:
# tifffile treats shape[0] as channel, need to expand to get labels displayed
#images = images[images.shape[0],np.newaxis,images.shape[1],images.shape[2]]
images = np.reshape(images,(images.shape[0],1,images.shape[1],images.shape[2]))
labels_list = []
if labels is None:
for i in range(images.shape[0]):
labels_list.append(label_prefix+str(i+1))
else:
labels_list = labels
ijtags = imagej_metadata_tags({'Labels':labels_list}, '<')
with tifffile.TiffWriter(path, bigtiff=False,imagej=True) as tif:
for i in range(images.shape[0]):
tif.save(images[i], metadata={'version':'1.11a','loop': False}, extratags=ijtags)
# Testing
if __name__ == "__main__":
def hamming_window(x,N):
y = 0.54 - 0.46*math.cos(2*math.pi*x/(N-1))
return y
hw = hamming_window
NT = 5
NX = 512
NY = 512
images = np.empty((NT,NY,NX),np.float32)
import time
print(str(time.time())+": Generating test images")
for t in range(NT):
images[t,:,:] = np.array([[(255-t*25)*hw(i,512)*hw(j,512) for i in range(NX)] for j in range(NY)],np.float32)
print(str(time.time())+": Test images generated")
print("Images shape: "+str(images.shape))
v = save("tiffwriter_test.tiff",images)
#!/usr/bin/env python3
__copyright__ = "Copyright 2018-2019, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
#python3 nn_ds_neibs30.py /data_ssd/lwir_sets/conf/qcstereo_lwir01.xml /data_ssd/lwir_sets/
#tensorboard --logdir="nn_ds_neibs30_graph13-9RNSWLAM0.5SLAM0.1SCLP0.2_nG_nI_HF_CP0.3_S0.03" --port=7001
import os
import sys
import numpy as np
import time
import shutil
from threading import Thread
import qcstereo_network
import qcstereo_losses
import qcstereo_functions as qsf
import tensorflow as tf
qsf.TIME_START = time.time()
qsf.TIME_LAST = qsf.TIME_START
IMG_WIDTH = 20 # 324 # tiles per image row
DEBUG_LEVEL= 1
try:
conf_file = sys.argv[1]
except IndexError:
print("Configuration path is required as a first argument. Optional second argument specifies root directory for data files")
exit(1)
try:
root_dir = sys.argv[2]
except IndexError:
root_dir = os.path.dirname(conf_file)
print ("Configuration file: " + conf_file)
parameters, dirs, files, _ = qsf.parseXmlConfig(conf_file, root_dir)
"""
Temporarily for backward compatibility
"""
if not "SLOSS_CLIP" in parameters:
parameters['SLOSS_CLIP'] = 0.5
print ("Old config, setting SLOSS_CLIP=", parameters['SLOSS_CLIP'])
"""
Defined in config file
"""
TILE_SIDE, TILE_LAYERS, TWO_TRAINS, NET_ARCH1, NET_ARCH2 = [None]*5
ABSOLUTE_DISPARITY,SYM8_SUB, WLOSS_LAMBDA, SLOSS_LAMBDA, SLOSS_CLIP = [None]*5
SPREAD_CONVERGENCE, INTER_CONVERGENCE, HOR_FLIP, DISP_DIFF_CAP, DISP_DIFF_SLOPE = [None]*5
CLUSTER_RADIUS = None
PARTIALS_WEIGHTS, MAX_IMGS_IN_MEM, MAX_FILES_PER_GROUP, BATCH_WEIGHTS, ONLY_TILE = [None] * 5
USE_CONFIDENCE, WBORDERS_ZERO, EPOCHS_TO_RUN, FILE_UPDATE_EPOCHS = [None] * 4
LR600,LR400,LR200,LR100,LR = [None]*5
SHUFFLE_FILES, EPOCHS_FULL_TEST, SAVE_TIFFS = [None] * 3
CHECKPOINT_PERIOD = None
TRAIN_BUFFER_GPU, TRAIN_BUFFER_CPU = [None]*2
TEST_TITLES = None
LOGFILE="results.txt"
"""
Next gets globals from the config file
"""
globals().update(parameters)
TRAIN_BUFFER_SIZE = TRAIN_BUFFER_GPU * TRAIN_BUFFER_CPU # in merged (quad) batches
#exit(0)
WIDTH = 20 # 324
HEIGHT = 15 # 242
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
BATCH_SIZE = ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SUFFIX=(str(NET_ARCH1)+'-'+str(NET_ARCH2)+
(["R","A"][ABSOLUTE_DISPARITY]) +
(["NS","S8"][SYM8_SUB])+
"WLAM"+str(WLOSS_LAMBDA)+
"SLAM"+str(SLOSS_LAMBDA)+
"SCLP"+str(SLOSS_CLIP)+
(['_nG','_G'][SPREAD_CONVERGENCE])+
(['_nI','_I'][INTER_CONVERGENCE]) +
(['_nHF',"_HF"][HOR_FLIP]) +
('_CP'+str(DISP_DIFF_CAP)) +
('_S'+str(DISP_DIFF_SLOPE))
)
NN_LAYOUT1 = qcstereo_network.NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = qcstereo_network.NN_LAYOUTS[NET_ARCH2]
USE_PARTIALS = not PARTIALS_WEIGHTS is None # False - just a single Siamese net, True - partial outputs that use concentric squares of the first level subnets
# Tiff export slice labels
SLICE_LABELS = ["nn_out_ext","hier_out_ext","gt_disparity","gt_strength",
"cutcorn_cost_nw","cutcorn_cost",
"gt-avg_dist","avg8_disp","gt_disp","out-avg"]
##############################################################################
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
qsf.prepareFiles(dirs, files, suffix = SUFFIX)
#copy config to results directory
print ("Copying config files to results directory:\n ('%s' -> '%s')"%(conf_file,dirs['result']))
try:
os.makedirs(dirs['result'])
except:
pass
shutil.copy2(conf_file,dirs['result'])
LOGPATH = os.path.join(dirs['result'],LOGFILE)
if TEST_TITLES is None:
TEST_TITLES = qsf.defaultTestTitles(files)
partials = None
partials = qsf.concentricSquares(CLUSTER_RADIUS)
PARTIALS_WEIGHTS = [1.0*pw/sum(PARTIALS_WEIGHTS) for pw in PARTIALS_WEIGHTS]
if not USE_PARTIALS:
partials = partials[0:1]
PARTIALS_WEIGHTS = [1.0]
qsf.evaluateAllResults(result_files = files['result'],
absolute_disparity = ABSOLUTE_DISPARITY,
cluster_radius = CLUSTER_RADIUS,
labels = SLICE_LABELS,
logpath= LOGPATH)
image_data = qsf.initImageData(
files = files,
max_imgs = MAX_IMGS_IN_MEM,
cluster_radius = CLUSTER_RADIUS,
tile_layers = TILE_LAYERS,
tile_side = TILE_SIDE,
width = IMG_WIDTH,
replace_nans = True)
corr2d_len, target_disparity_len, gtds_len, _ = qsf.get_lengths(CLUSTER_RADIUS, TILE_LAYERS, TILE_SIDE)
train_next, dataset_train, datasets_test= qsf.initTrainTestData(
files = files,
cluster_radius = CLUSTER_RADIUS,
buffer_size = TRAIN_BUFFER_SIZE * BATCH_SIZE, # number of clusters per train
test_titles = TEST_TITLES)
corr2d_train_placeholder = tf.compat.v1.placeholder(dataset_train.dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.compat.v1.placeholder(dataset_train.dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.compat.v1.placeholder(dataset_train.dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d": corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
tf_batch_weights = tf.compat.v1.placeholder(shape=(None,), dtype=tf.float32, name = "batch_weights") # way to increase importance of the high variance clusters
feed_batch_weights = np.array(BATCH_WEIGHTS*(BATCH_SIZE//len(BATCH_WEIGHTS)), dtype=np.float32)
feed_batch_weight_1 = np.array([1.0], dtype=np.float32)
dataset_test_size = len(datasets_test[0])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(image_data[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
target_disparity_cluster = tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
target_disparity_cluster], axis=2, name = "corr2d_Nx325")
if SPREAD_CONVERGENCE:
outs, inp_weights = qcstereo_network.networks_siam(
input = corr2d_Nx325,
input_global = target_disparity_cluster,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
inter_convergence = INTER_CONVERGENCE,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE, #Remove/put None for normal operation
partials = partials,
use_confidence= USE_CONFIDENCE,
cluster_radius = CLUSTER_RADIUS)
else:
outs, inp_weights = qcstereo_network.networks_siam(
input_tensor= corr2d_Nx325,
input_global = None,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
inter_convergence = False,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE, #Remove/put None for normal operation
partials = partials,
use_confidence= USE_CONFIDENCE,
cluster_radius = CLUSTER_RADIUS)
tf_partial_weights = tf.constant(PARTIALS_WEIGHTS,dtype=tf.float32,name="partial_weights")
G_losses = [0.0]*len(partials)
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1]
gt_ds_batch_clust = next_element_tt['gt_ds']
gt_ds_batch = gt_ds_batch_clust[:,2 * center_tile_index: 2 * (center_tile_index +1)]
G_losses[0], _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = qcstereo_losses.batchLoss(
out_batch = outs[0], # [batch_size,(1..2)] tf_result
target_disparity_batch= target_disparity_batch, # next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = gt_ds_batch, # next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
batch_weights = tf_batch_weights,
disp_diff_cap = DISP_DIFF_CAP,
disp_diff_slope= DISP_DIFF_SLOPE,
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
## lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
G_loss = G_losses[0]
for n in range (1,len(partials)):
G_losses[n], _, _, _, _, _, _, _ = qcstereo_losses.batchLoss(
out_batch = outs[n], # [batch_size,(1..2)] tf_result
target_disparity_batch= target_disparity_batch, #next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = gt_ds_batch, # next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
batch_weights = tf_batch_weights,
disp_diff_cap = DISP_DIFF_CAP,
disp_diff_slope= DISP_DIFF_SLOPE,
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
# lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
tf_wlosses = tf.multiply(G_losses, tf_partial_weights, name = "tf_wlosses")
G_losses_sum = tf.reduce_sum(tf_wlosses, name = "G_losses_sum")
if SLOSS_LAMBDA > 0:
S_loss, rslt_cost_nw, rslt_cost_w, rslt_d , rslt_avg_disparity, rslt_gt_disparity, rslt_offs = qcstereo_losses.smoothLoss(
out_batch = outs[0], # [batch_size,(1..2)] tf_result
target_disparity_batch = target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch_clust = gt_ds_batch_clust, # [batch_size,25,2] tf placeholder
clip = SLOSS_CLIP,
absolute_disparity = ABSOLUTE_DISPARITY, #when false there should be no activation on disparity output !
cluster_radius = CLUSTER_RADIUS)
GS_loss = tf.add(G_losses_sum, SLOSS_LAMBDA * S_loss, name = "GS_loss")
else:
S_loss = tf.constant(0.0, dtype=tf.float32,name = "S_loss")
GS_loss = G_losses_sum # G_loss
if WLOSS_LAMBDA > 0.0:
W_loss = qcstereo_losses.weightsLoss(
inp_weights = inp_weights[0], # inp_weights - list of tensors, currently - just [0]
tile_layers= TILE_LAYERS, # 4
tile_side = TILE_SIDE, # 9
wborders_zero = WBORDERS_ZERO)
GW_loss = tf.add(GS_loss, WLOSS_LAMBDA * W_loss, name = "GW_loss")
else:
GW_loss = GS_loss # G_loss
W_loss = tf.constant(0.0, dtype=tf.float32,name = "W_loss")
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_G_losses = tf.compat.v1.placeholder(tf.float32,shape=[len(partials)],name='G_losses_avg')
tf_ph_S_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='S_loss_avg')
tf_ph_W_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='W_loss_avg')
tf_ph_GW_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='GW_loss_avg')
tf_ph_sq_diff = tf.compat.v1.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.compat.v1.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.compat.v1.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.compat.v1.placeholder(tf.float32,shape=None,name='img_test9')
"""
with tf.name_scope('sample'):
tf.compat.v1.summary.scalar("GW_loss", GW_loss)
tf.compat.v1.summary.scalar("G_loss", G_loss)
tf.compat.v1.summary.scalar("S_loss", S_loss)
tf.compat.v1.summary.scalar("W_loss", W_loss)
tf.compat.v1.summary.scalar("sq_diff", _cost1)
tf.compat.v1.summary.scalar("gtvar_diff", GT_variance)
"""
with tf.name_scope('epoch_average'):
for i in range(tf_ph_G_losses.shape[0]):
tf.compat.v1.summary.scalar("G_loss_epoch_"+str(i), tf_ph_G_losses[i])
tf.compat.v1.summary.scalar("GW_loss_epoch", tf_ph_GW_loss)
tf.compat.v1.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.compat.v1.summary.scalar("S_loss_epoch", tf_ph_S_loss)
tf.compat.v1.summary.scalar("W_loss_epoch", tf_ph_W_loss)
tf.compat.v1.summary.scalar("sq_diff_epoch", tf_ph_sq_diff)
tf.compat.v1.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.compat.v1.summary.scalar("img_test0", tf_img_test0)
tf.compat.v1.summary.scalar("img_test9", tf_img_test9)
trainable_vars= tf.trainable_variables()
lr= tf.compat.v1.placeholder(tf.float32)
G_opt= tf.compat.v1.train.AdamOptimizer(learning_rate=lr).minimize(GW_loss)
ROOT_PATH = './attic/nn_ds_neibs30_graph'+SUFFIX+"/" # for tensorboard
TT_SUBDIRS = ['train'] #,'test0','test1','test2','test3']
for i,_ in enumerate(datasets_test):
# TT_SUBDIRS.append('test%d'%(i))
TT_SUBDIRS.append(TEST_TITLES[i].replace(' ','_'))
TT_PATHS = [ROOT_PATH + p for p in TT_SUBDIRS]
# CLEAN OLD STAFF
shutil.rmtree(ROOT_PATH, ignore_errors=True)
#for p in TT_PATHS:
# shutil.rmtree(p, ignore_errors=True)
#seems that runs use directory creation time to order graphs
#for p in TT_PATHS:
# os.makedirs(p)
# time.sleep(1.5) # reduce later
num_train_subs = len(train_next) # number of (different type) merged training sets
dataset_train_size = TRAIN_BUFFER_GPU * num_train_subs # TRAIN_BUFFER_SIZE
tt_summaries = [0.0 for e in TT_SUBDIRS]
tt2_avg = [0.0 for e in TT_SUBDIRS]
tt_gw_avg = [0.0 for e in TT_SUBDIRS]
tt_g_avgs = [[0.0]*len(partials) for e in TT_SUBDIRS]
tt_w_avg = [0.0 for e in TT_SUBDIRS]
tt_s_avg = [0.0 for e in TT_SUBDIRS]
tt_gtvar_avg = [0.0 for e in TT_SUBDIRS]
saver=tf.compat.v1.train.Saver(trainable_vars)
saver_def = saver.as_saver_def()
# The name of the tensor you must feed with a filename when saving/restoring.
print ('saver_def.filename_tensor_name=',saver_def.filename_tensor_name)
# The name of the target operation you must run when restoring.
print ('saver_def.restore_op_name=',saver_def.restore_op_name)
# The name of the target operation you must run when saving.
print ('saver_def.save_tensor_name=',saver_def.save_tensor_name)
try:
os.makedirs(os.path.dirname(files['checkpoints']))
print ("Created directory ",os.path.dirname(files['checkpoints']))
# os.makedirs(files['checkpoints'])
except:
pass
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
tt_writers = []
for p in TT_PATHS:
tt_writers.append(tf.summary.FileWriter(p, sess.graph))
print ("Adding delay to make directory creation time different: "+p)
time.sleep(2.0) # reduce later
loss_gw_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_g_train_hists= [np.empty(dataset_train_size, dtype=np.float32) for p in partials]
loss_s_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_w_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_gw_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_g_test_hists= [np.empty(dataset_test_size, dtype=np.float32) for p in partials]
loss_s_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_w_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
thr=None
thr_result = None
trains_to_update = [train_next[n_train]['more_files'] for n_train in range(len(train_next))]
for epoch in range (EPOCHS_TO_RUN):
"""
update files after each epoch, all 4.
Convert to threads after testing
"""
if (FILE_UPDATE_EPOCHS > 0) and (epoch % FILE_UPDATE_EPOCHS == 0):
if not thr is None:
if thr.is_alive():
qsf.print_time("***WAITING*** until tfrecord gets loaded", end=" ")
else:
qsf.print_time("tfrecord is ***ALREADY LOADED*** ", end=" ")
thr.join()
qsf.print_time("Done")
qsf.print_time("Inserting new data", end=" ")
for n_train in range(len(trains_to_update)):
if trains_to_update[n_train]:
qsf.add_file_to_dataset(dataset = dataset_train,
new_dataset = thr_result[n_train],
train_next = train_next[n_train])
qsf.print_time("Done")
thr_result = []
fpaths = []
for n_train in range(len(trains_to_update)):
if trains_to_update[n_train]:
fpaths.append(files['train'][n_train][train_next[n_train]['file']])
qsf.print_time("Will read in background: "+fpaths[-1])
thr = Thread(target=qsf.getMoreFiles, args=(fpaths,thr_result, CLUSTER_RADIUS, HOR_FLIP, TILE_LAYERS, TILE_SIDE))
thr.start()
train_buf_index = epoch % TRAIN_BUFFER_CPU # GPU memory from CPU memory (now 4)
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
if (train_buf_index == 0) and SHUFFLE_FILES:
qsf.print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
qsf.shuffle_in_place(
dataset_data = dataset_train, #alternating clusters from 4 sources.each cluster has all needed data (concatenated)
period = num_train_subs)
qsf.print_time(" Done")
sti = train_buf_index * dataset_train_size * BATCH_SIZE # TRAIN_BUFFER_GPU * num_train_subs
eti = sti+ dataset_train_size * BATCH_SIZE# (train_buf_index +1) * TRAIN_BUFFER_GPU * num_train_subs
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_train[sti:eti,:corr2d_len],
target_disparity_train_placeholder: dataset_train[sti:eti,corr2d_len:corr2d_len+target_disparity_len],
gt_ds_train_placeholder: dataset_train[sti:eti,corr2d_len+target_disparity_len:corr2d_len+target_disparity_len+gtds_len] })
for i in range(dataset_train_size):
# try:
_, GW_loss_trained, G_losses_trained, S_loss_trained, W_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ G_opt,
GW_loss,
G_losses,
S_loss,
W_loss,
outs[0],
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={tf_batch_weights: feed_batch_weights,
lr: learning_rate
}) # previous value of *_avg #Fetch argument 0.0 has invalid type <class 'float'>, must be a string or Tensor. (Can not convert a float into a Tensor or Operation.)
loss_gw_train_hist[i] = GW_loss_trained
for nn, gl in enumerate(G_losses_trained):
loss_g_train_hists[nn][i] = gl
loss_s_train_hist[i] = S_loss_trained
loss_w_train_hist[i] = W_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
# except tf.errors.OutOfRangeError:
# print("****** NO MORE DATA! train done at step %d"%(i))
# break
tt_gw_avg[0] = np.average(loss_gw_train_hist).astype(np.float32)
for nn, lgth in enumerate(loss_g_train_hists):
tt_g_avgs[0][nn] = np.average(lgth).astype(np.float32)
tt_s_avg[0] = np.average(loss_s_train_hist).astype(np.float32)
tt_w_avg[0] = np.average(loss_w_train_hist).astype(np.float32)
tt2_avg[0] = np.average(loss2_train_hist).astype(np.float32)
tt_gtvar_avg[0] = np.average(gtvar_train_hist).astype(np.float32)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test[:, :corr2d_len], #['corr2d'],
target_disparity_train_placeholder: dataset_test[:, corr2d_len:corr2d_len+target_disparity_len], # ['target_disparity'],
gt_ds_train_placeholder: dataset_test[:, corr2d_len+target_disparity_len:] }) # ['gt_ds']})
"""
TODO: Make it possible to have different length dataset_test arrays to mix different length test files
"""
for i in range(dataset_test_size):
# for i in range(dataset_test.shape[0]):
# try:
GW_loss_tested, G_losses_tested, S_loss_tested, W_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[GW_loss,
G_losses,
S_loss,
W_loss,
outs[0],
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={tf_batch_weights: feed_batch_weight_1 , # feed_batch_weights,
lr: learning_rate
}) # previous value of *_avg
loss_gw_test_hist[i] = GW_loss_tested
for nn, gl in enumerate(G_losses_tested):
loss_g_test_hists[nn][i] = gl
loss_s_test_hist[i] = S_loss_tested
loss_w_test_hist[i] = W_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
# except tf.errors.OutOfRangeError:
# print("test done at step %d"%(i))
# break
tt_gw_avg[ntest+1] = np.average(loss_gw_test_hist).astype(np.float32)
for nn, lgth in enumerate(loss_g_test_hists):
tt_g_avgs[ntest+1][nn] = np.average(lgth).astype(np.float32)
tt_s_avg[ntest+1] = np.average(loss_s_test_hist).astype(np.float32)
tt_w_avg[ntest+1] = np.average(loss_w_test_hist).astype(np.float32)
tt2_avg[ntest+1] = np.average(loss2_test_hist).astype(np.float32)
tt_gtvar_avg[ntest+1] = np.average(gtvar_test_hist).astype(np.float32)
if (((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0)) and (len(image_data) > 0) :
lf = None
if (epoch + 1) == EPOCHS_TO_RUN: # last
print("Last epoch, removing train/test datasets to reduce memory footprint")
del(dataset_train)
del(dataset_test)
if LOGPATH:
lf=open(LOGPATH,"w") #overwrite previous (or make it "a"?
last_epoch = (epoch + 1) == EPOCHS_TO_RUN
ind_img = [0]
if last_epoch:
ind_img = [i for i in range(len(image_data))]
###################################################
# Read the full image
###################################################
## test_summaries_img = [0.0]*len(ind_img) # datasets_img)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_cost_nw= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_cost_w= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_d= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_avg_disparity = np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_gt_disparity = np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_offs = np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest in ind_img: # datasets_img):
dataset_img = qsf.readImageData(
image_data = image_data,
files = files,
indx = ntest,
cluster_radius = CLUSTER_RADIUS,
tile_layers = TILE_LAYERS,
tile_side = TILE_SIDE,
width = IMG_WIDTH,
replace_nans = True)
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
# try:
output, cost_nw, cost_w, dd, avg_disparity, gt_disparity, offs = sess.run(
[outs[0], # {?,1]
rslt_cost_nw, #[?,]
rslt_cost_w, #[?,]
rslt_d, #[?,]
rslt_avg_disparity,
rslt_gt_disparity,
rslt_offs
],
feed_dict={
tf_batch_weights: feed_batch_weight_1 # feed_batch_weights,
}) # previous value of *_avg
# except tf.errors.OutOfRangeError:
# print("test done at step %d"%(i))
# break
# try:
disp_out[start_offs:end_offs] = output.flatten()
dbg_cost_nw[start_offs:end_offs] = cost_nw.flatten()
dbg_cost_w [start_offs:end_offs] = cost_w.flatten()
dbg_d[start_offs:end_offs] = dd.flatten()
dbg_avg_disparity[start_offs:end_offs] = avg_disparity.flatten()
dbg_gt_disparity[start_offs:end_offs] = gt_disparity.flatten()
dbg_offs[start_offs:end_offs] = offs.flatten()
# except ValueError:
# print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
# break;
pass
result_file = files['result'][ntest] # result_files[ntest]
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate(
[disp_out.reshape(-1,1),
dataset_img['t_disps'], #t_disps[ntest],
dataset_img['gtruths'], # gtruths[ntest],
dbg_cost_nw.reshape(-1,1),
dbg_cost_w.reshape(-1,1),
dbg_d.reshape(-1,1),
dbg_avg_disparity.reshape(-1,1),
dbg_gt_disparity.reshape(-1,1),
dbg_offs.reshape(-1,1)],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = qsf.eval_results(result_file, ABSOLUTE_DISPARITY, radius=CLUSTER_RADIUS, logfile=lf)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
if SAVE_TIFFS:
qsf.result_npy_to_tiff(result_file, ABSOLUTE_DISPARITY, fix_nan = True,labels=SLICE_LABELS, logfile=lf)
"""
Remove dataset_img (if it is not [0] to reduce memory footprint
"""
if ntest > 0:
image_data[ntest] = None
if lf:
lf.close()
# tensorboard scalars
tt_summaries[0] = sess.run([merged],
feed_dict={ tf_ph_GW_loss: tt_gw_avg[0],
tf_ph_G_loss: tt_g_avgs[0][0], #train_g_avg,
tf_ph_G_losses: tt_g_avgs[0],
tf_ph_S_loss: tt_s_avg[0],
tf_ph_W_loss: tt_w_avg[0],
tf_ph_sq_diff: tt2_avg[0], # train2_avg,
tf_gtvar_diff: tt_gtvar_avg[0],
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9}) # previous value of *_avg #Fetch argument 0.0 has invalid type <class 'float'>, must be a string or Tensor. (Can not convert a float into a Tensor or Operation.)
for ntest, _ in enumerate(datasets_test):
tt_summaries[ntest+1] = sess.run([merged],
feed_dict={
tf_ph_GW_loss: tt_gw_avg[ntest+1],
tf_ph_G_loss: tt_g_avgs[ntest+1][0],
tf_ph_G_losses: tt_g_avgs[ntest+1], # train_g_avgs, # temporary, there is o data fro test
tf_ph_S_loss: tt_s_avg[ntest+1],
tf_ph_W_loss: tt_w_avg[ntest+1],
tf_ph_sq_diff: tt2_avg[ntest+1], #test2_avg,
tf_gtvar_diff: tt_gtvar_avg[ntest+1],
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9})
for n,tt_writer in enumerate(tt_writers):
## tt_writer.add_summary(tt_summaries[n],epoch)
tt_writer.add_summary(tt_summaries[n][0],epoch)
# if epoch ==0 :
# print ("adding delay to make directory creation time different")
# time.sleep(2.0) # reduce later
qsf.print_time("==== %04d:%03d -> %.4f %.4f %.4f %.4f %.4f (%.4f %.4f %.4f %.4f %.4f) ===="%(
epoch,i,
tt_gw_avg[0], tt_gw_avg[1], tt_gw_avg[2], tt_gw_avg[3], tt_gw_avg[4],
tt2_avg[0], tt2_avg[1], tt2_avg[2], tt2_avg[3], tt2_avg[4]))
if (not CHECKPOINT_PERIOD is None) and (((epoch + 1) % CHECKPOINT_PERIOD) == 0):
print("Saving periodic checkpoint (trained variables only) to %s, global_step = %d"%(os.path.dirname(files['checkpoints']), epoch),end=" => ")
print(saver.save(sess, files['checkpoints'], global_step=epoch, write_meta_graph=False))
# Close writers
for tt_writer in tt_writers:
try:
tt_writer.close()
except:
print ("Could not close tt_writer: ",tt_writer)
print("Saving final checkpoint (trained variables only) to %s"%(files['checkpoints']),end=" => ")
print(saver.save(sess, files["checkpoints"]))
print("All done")
exit (0)
"""
Traceback (most recent call last):
File "nn_ds_neibs30.py", line 721, in <module>
tt2_avg[0], tt2_avg[1], tt2_avg[2], tt2_avg[3], tt2_avg[4]))
ValueError: unsupported format character ' ' (0x20) at index 20
"""
\ No newline at end of file
#!/usr/bin/env python3
__copyright__ = "Copyright 2018-2019, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
#python3 nn_ds_neibs31.py /data_ssd/lwir_sets/conf/qcstereo_lwir05.xml /data_ssd/lwir_sets/
#tensorboard --logdir="nn_ds_neibs30_graph13-9RNSWLAM0.5SLAM0.1SCLP0.2_nG_nI_HF_CP0.3_S0.03" --port=7001
import os
import sys
import numpy as np
import time
import shutil
from threading import Thread
import qcstereo_network
import qcstereo_losses
import qcstereo_functions as qsf
import tensorflow as tf
qsf.TIME_START = time.time()
qsf.TIME_LAST = qsf.TIME_START
IMG_WIDTH = 20 # 324 # tiles per image row
DEBUG_LEVEL= 1
try:
conf_file = sys.argv[1]
except IndexError:
print("Configuration path is required as a first argument. Optional second argument specifies root directory for data files")
exit(1)
try:
root_dir = sys.argv[2]
except IndexError:
root_dir = os.path.dirname(conf_file)
print ("Configuration file: " + conf_file)
parameters, dirs, files, _ = qsf.parseXmlConfig(conf_file, root_dir)
"""
Temporarily for backward compatibility
"""
if not "SLOSS_CLIP" in parameters:
parameters['SLOSS_CLIP'] = 0.5
print ("Old config, setting SLOSS_CLIP=", parameters['SLOSS_CLIP'])
"""
Defined in config file
"""
TILE_SIDE, TILE_LAYERS, TWO_TRAINS, NET_ARCH1, NET_ARCH2 = [None]*5
ABSOLUTE_DISPARITY,SYM8_SUB, WLOSS_LAMBDA, SLOSS_LAMBDA, SLOSS_CLIP = [None]*5
SPREAD_CONVERGENCE, INTER_CONVERGENCE, HOR_FLIP, DISP_DIFF_CAP, DISP_DIFF_SLOPE = [None]*5
CLUSTER_RADIUS = None
PARTIALS_WEIGHTS, MAX_IMGS_IN_MEM, MAX_FILES_PER_GROUP, BATCH_WEIGHTS, ONLY_TILE = [None] * 5
USE_CONFIDENCE, WBORDERS_ZERO, EPOCHS_TO_RUN, FILE_UPDATE_EPOCHS = [None] * 4
LR600,LR400,LR200,LR100,LR = [None]*5
SHUFFLE_FILES, EPOCHS_FULL_TEST, SAVE_TIFFS = [None] * 3
CHECKPOINT_PERIOD = None
TRAIN_BUFFER_GPU, TRAIN_BUFFER_CPU = [None]*2
TEST_TITLES = None
LOGFILE="results.txt"
"""
Next gets globals from the config file
"""
globals().update(parameters)
TRAIN_BUFFER_SIZE = TRAIN_BUFFER_GPU * TRAIN_BUFFER_CPU # in merged (quad) batches
#exit(0)
WIDTH = 20 # 324
HEIGHT = 15 # 242
TILE_SIZE = TILE_SIDE* TILE_SIDE # == 81
FEATURES_PER_TILE = TILE_LAYERS * TILE_SIZE# == 324
BATCH_SIZE = ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
SUFFIX=(str(NET_ARCH1)+'-'+str(NET_ARCH2)+
(["R","A"][ABSOLUTE_DISPARITY]) +
(["NS","S8"][SYM8_SUB])+
"WLAM"+str(WLOSS_LAMBDA)+
"SLAM"+str(SLOSS_LAMBDA)+
"SCLP"+str(SLOSS_CLIP)+
(['_nG','_G'][SPREAD_CONVERGENCE])+
(['_nI','_I'][INTER_CONVERGENCE]) +
(['_nHF',"_HF"][HOR_FLIP]) +
('_CP'+str(DISP_DIFF_CAP)) +
('_S'+str(DISP_DIFF_SLOPE))
)
NN_LAYOUT1 = qcstereo_network.NN_LAYOUTS[NET_ARCH1]
NN_LAYOUT2 = qcstereo_network.NN_LAYOUTS[NET_ARCH2]
USE_PARTIALS = not PARTIALS_WEIGHTS is None # False - just a single Siamese net, True - partial outputs that use concentric squares of the first level subnets
# Tiff export slice labels
SLICE_LABELS = ["nn_out_ext","hier_out_ext","gt_disparity","gt_strength",
"cutcorn_cost_nw","cutcorn_cost",
"gt-avg_dist","avg8_disp","gt_disp","out-avg"]
##############################################################################
cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
qsf.prepareFiles(dirs, files, suffix = SUFFIX)
#copy config to results directory
print ("Copying config files to results directory:\n ('%s' -> '%s')"%(conf_file,dirs['result']))
try:
os.makedirs(dirs['result'])
except:
pass
shutil.copy2(conf_file,dirs['result'])
LOGPATH = os.path.join(dirs['result'],LOGFILE)
if TEST_TITLES is None:
TEST_TITLES = qsf.defaultTestTitles(files)
partials = None
partials = qsf.concentricSquares(CLUSTER_RADIUS)
PARTIALS_WEIGHTS = [1.0*pw/sum(PARTIALS_WEIGHTS) for pw in PARTIALS_WEIGHTS]
if not USE_PARTIALS:
partials = partials[0:1]
PARTIALS_WEIGHTS = [1.0]
qsf.evaluateAllResults(result_files = files['result'],
absolute_disparity = ABSOLUTE_DISPARITY,
cluster_radius = CLUSTER_RADIUS,
labels = SLICE_LABELS,
logpath= LOGPATH)
image_data = qsf.initImageData(
files = files,
max_imgs = MAX_IMGS_IN_MEM,
cluster_radius = CLUSTER_RADIUS,
tile_layers = TILE_LAYERS,
tile_side = TILE_SIDE,
width = IMG_WIDTH,
replace_nans = True)
corr2d_len, target_disparity_len, gtds_len, _ = qsf.get_lengths(CLUSTER_RADIUS, TILE_LAYERS, TILE_SIDE)
train_next, dataset_train, datasets_test= qsf.initTrainTestData(
files = files,
cluster_radius = CLUSTER_RADIUS,
buffer_size = TRAIN_BUFFER_SIZE * BATCH_SIZE, # number of clusters per train
test_titles = TEST_TITLES)
corr2d_train_placeholder = tf.compat.v1.placeholder(dataset_train.dtype, (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
target_disparity_train_placeholder = tf.compat.v1.placeholder(dataset_train.dtype, (None,1 * cluster_size)) #target_disparity_train.shape)
gt_ds_train_placeholder = tf.compat.v1.placeholder(dataset_train.dtype, (None,2 * cluster_size)) #gt_ds_train.shape)
dataset_tt = tf.data.Dataset.from_tensor_slices({
"corr2d": corr2d_train_placeholder,
"target_disparity": target_disparity_train_placeholder,
"gt_ds": gt_ds_train_placeholder})
tf_batch_weights = tf.compat.v1.placeholder(shape=(None,), dtype=tf.float32, name = "batch_weights") # way to increase importance of the high variance clusters
feed_batch_weights = np.array(BATCH_WEIGHTS*(BATCH_SIZE//len(BATCH_WEIGHTS)), dtype=np.float32)
feed_batch_weight_1 = np.array([1.0], dtype=np.float32)
dataset_test_size = len(datasets_test[0])
dataset_test_size //= BATCH_SIZE
dataset_img_size = len(image_data[0]['corr2d'])
dataset_img_size //= BATCH_SIZE
dataset_tt = dataset_tt.batch(BATCH_SIZE)
dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
iterator_tt = dataset_tt.make_initializable_iterator()
next_element_tt = iterator_tt.get_next()
result_dir = './attic/result_neibs_'+ SUFFIX+'/'
checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
save_freq = 500
def debug_gt_variance(
indx, # This tile index (0..8)
center_indx, # center tile index
gt_ds_batch # [?:9:2]
):
with tf.name_scope("Debug_GT_Variance"):
d_gt_this = tf.reshape(gt_ds_batch[:,2 * indx],[-1], name = "d_this")
d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1], name = "d_center")
d_gt_diff = tf.subtract(d_gt_this, d_gt_center, name = "d_diff")
d_gt_diff2 = tf.multiply(d_gt_diff, d_gt_diff, name = "d_diff2")
d_gt_var = tf.reduce_mean(d_gt_diff2, name = "d_gt_var")
return d_gt_var
target_disparity_cluster = tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")
corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
target_disparity_cluster], axis=2, name = "corr2d_Nx325")
if SPREAD_CONVERGENCE:
outs, inp_weights = qcstereo_network.networks_siam(
input = corr2d_Nx325,
input_global = target_disparity_cluster,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
inter_convergence = INTER_CONVERGENCE,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE, #Remove/put None for normal operation
partials = partials,
use_confidence= USE_CONFIDENCE,
cluster_radius = CLUSTER_RADIUS)
else:
outs, inp_weights = qcstereo_network.networks_siam(
input_tensor= corr2d_Nx325,
input_global = None,
layout1 = NN_LAYOUT1,
layout2 = NN_LAYOUT2,
inter_convergence = False,
sym8 = SYM8_SUB,
only_tile = ONLY_TILE, #Remove/put None for normal operation
partials = partials,
use_confidence= USE_CONFIDENCE,
cluster_radius = CLUSTER_RADIUS)
tf_partial_weights = tf.constant(PARTIALS_WEIGHTS,dtype=tf.float32,name="partial_weights")
G_losses = [0.0]*len(partials)
target_disparity_batch= next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1]
gt_ds_batch_clust = next_element_tt['gt_ds']
gt_ds_batch = gt_ds_batch_clust[:,2 * center_tile_index: 2 * (center_tile_index +1)]
G_losses[0], _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = qcstereo_losses.batchLoss(
out_batch = outs[0], # [batch_size,(1..2)] tf_result
target_disparity_batch= target_disparity_batch, # next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = gt_ds_batch, # next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
batch_weights = tf_batch_weights,
disp_diff_cap = DISP_DIFF_CAP,
disp_diff_slope= DISP_DIFF_SLOPE,
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
## lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
G_loss = G_losses[0]
for n in range (1,len(partials)):
G_losses[n], _, _, _, _, _, _, _ = qcstereo_losses.batchLoss(
out_batch = outs[n], # [batch_size,(1..2)] tf_result
target_disparity_batch= target_disparity_batch, #next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d, # [batch_size] tf placeholder
gt_ds_batch = gt_ds_batch, # next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)], # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt, # [batch_size,2] tf placeholder
batch_weights = tf_batch_weights,
disp_diff_cap = DISP_DIFF_CAP,
disp_diff_slope= DISP_DIFF_SLOPE,
absolute_disparity = ABSOLUTE_DISPARITY,
use_confidence = USE_CONFIDENCE, # True,
lambda_conf_avg = 0.01,
# lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 2.0,
error2_offset = 0, # 0.0025, # (0.05^2)
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False) # use calculated disparity for disparity weight boosting (False - use target disparity)
tf_wlosses = tf.multiply(G_losses, tf_partial_weights, name = "tf_wlosses")
G_losses_sum = tf.reduce_sum(tf_wlosses, name = "G_losses_sum")
if SLOSS_LAMBDA > 0:
S_loss, rslt_cost_nw, rslt_cost_w, rslt_d , rslt_avg_disparity, rslt_gt_disparity, rslt_offs = qcstereo_losses.smoothLoss(
out_batch = outs[0], # [batch_size,(1..2)] tf_result
target_disparity_batch = target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch_clust = gt_ds_batch_clust, # [batch_size,25,2] tf placeholder
clip = SLOSS_CLIP,
absolute_disparity = ABSOLUTE_DISPARITY, #when false there should be no activation on disparity output !
cluster_radius = CLUSTER_RADIUS)
GS_loss = tf.add(G_losses_sum, SLOSS_LAMBDA * S_loss, name = "GS_loss")
else:
S_loss = tf.constant(0.0, dtype=tf.float32,name = "S_loss")
GS_loss = G_losses_sum # G_loss
if WLOSS_LAMBDA > 0.0:
W_loss = qcstereo_losses.weightsLoss(
inp_weights = inp_weights[0], # inp_weights - list of tensors, currently - just [0]
tile_layers= TILE_LAYERS, # 4
tile_side = TILE_SIDE, # 9
wborders_zero = WBORDERS_ZERO)
GW_loss = tf.add(GS_loss, WLOSS_LAMBDA * W_loss, name = "GW_loss")
else:
GW_loss = GS_loss # G_loss
W_loss = tf.constant(0.0, dtype=tf.float32,name = "W_loss")
GT_variance = debug_gt_variance(indx = 0, # This tile index (0..8)
center_indx = 4, # center tile index
gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
tf_ph_G_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='G_loss_avg')
tf_ph_G_losses = tf.compat.v1.placeholder(tf.float32,shape=[len(partials)],name='G_losses_avg')
tf_ph_S_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='S_loss_avg')
tf_ph_W_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='W_loss_avg')
tf_ph_GW_loss = tf.compat.v1.placeholder(tf.float32,shape=None,name='GW_loss_avg')
tf_ph_sq_diff = tf.compat.v1.placeholder(tf.float32,shape=None,name='sq_diff_avg')
tf_gtvar_diff = tf.compat.v1.placeholder(tf.float32,shape=None,name='gtvar_diff')
tf_img_test0 = tf.compat.v1.placeholder(tf.float32,shape=None,name='img_test0')
tf_img_test9 = tf.compat.v1.placeholder(tf.float32,shape=None,name='img_test9')
"""
with tf.name_scope('sample'):
tf.compat.v1.summary.scalar("GW_loss", GW_loss)
tf.compat.v1.summary.scalar("G_loss", G_loss)
tf.compat.v1.summary.scalar("S_loss", S_loss)
tf.compat.v1.summary.scalar("W_loss", W_loss)
tf.compat.v1.summary.scalar("sq_diff", _cost1)
tf.compat.v1.summary.scalar("gtvar_diff", GT_variance)
"""
with tf.name_scope('epoch_average'):
for i in range(tf_ph_G_losses.shape[0]):
tf.compat.v1.summary.scalar("G_loss_epoch_"+str(i), tf_ph_G_losses[i])
tf.compat.v1.summary.scalar("GW_loss_epoch", tf_ph_GW_loss)
tf.compat.v1.summary.scalar("G_loss_epoch", tf_ph_G_loss)
tf.compat.v1.summary.scalar("S_loss_epoch", tf_ph_S_loss)
tf.compat.v1.summary.scalar("W_loss_epoch", tf_ph_W_loss)
tf.compat.v1.summary.scalar("sq_diff_epoch", tf_ph_sq_diff)
tf.compat.v1.summary.scalar("gtvar_diff", tf_gtvar_diff)
tf.compat.v1.summary.scalar("img_test0", tf_img_test0)
tf.compat.v1.summary.scalar("img_test9", tf_img_test9)
trainable_vars= tf.trainable_variables()
lr= tf.compat.v1.placeholder(tf.float32)
G_opt= tf.compat.v1.train.AdamOptimizer(learning_rate=lr).minimize(GW_loss)
ROOT_PATH = './attic/nn_ds_neibs30_graph'+SUFFIX+"/" # for tensorboard
TT_SUBDIRS = ['train'] #,'test0','test1','test2','test3']
for i,_ in enumerate(datasets_test):
# TT_SUBDIRS.append('test%d'%(i))
TT_SUBDIRS.append(TEST_TITLES[i].replace(' ','_'))
TT_PATHS = [ROOT_PATH + p for p in TT_SUBDIRS]
# CLEAN OLD STAFF
shutil.rmtree(ROOT_PATH, ignore_errors=True)
#for p in TT_PATHS:
# shutil.rmtree(p, ignore_errors=True)
#seems that runs use directory creation time to order graphs
#for p in TT_PATHS:
# os.makedirs(p)
# time.sleep(1.5) # reduce later
num_train_subs = len(train_next) # number of (different type) merged training sets
dataset_train_size = TRAIN_BUFFER_GPU * num_train_subs # TRAIN_BUFFER_SIZE
tt_summaries = [0.0 for e in TT_SUBDIRS]
tt2_avg = [0.0 for e in TT_SUBDIRS]
tt_gw_avg = [0.0 for e in TT_SUBDIRS]
tt_g_avgs = [[0.0]*len(partials) for e in TT_SUBDIRS]
tt_w_avg = [0.0 for e in TT_SUBDIRS]
tt_s_avg = [0.0 for e in TT_SUBDIRS]
tt_gtvar_avg = [0.0 for e in TT_SUBDIRS]
saver=tf.compat.v1.train.Saver(trainable_vars)
saver_def = saver.as_saver_def()
# The name of the tensor you must feed with a filename when saving/restoring.
print ('saver_def.filename_tensor_name=',saver_def.filename_tensor_name)
# The name of the target operation you must run when restoring.
print ('saver_def.restore_op_name=',saver_def.restore_op_name)
# The name of the target operation you must run when saving.
print ('saver_def.save_tensor_name=',saver_def.save_tensor_name)
try:
os.makedirs(os.path.dirname(files['checkpoints']))
print ("Created directory ",os.path.dirname(files['checkpoints']))
# os.makedirs(files['checkpoints'])
except:
pass
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
merged = tf.summary.merge_all()
tt_writers = []
for p in TT_PATHS:
tt_writers.append(tf.summary.FileWriter(p, sess.graph))
print ("Adding delay to make directory creation time different: "+p)
time.sleep(2.0) # reduce later
loss_gw_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_g_train_hists= [np.empty(dataset_train_size, dtype=np.float32) for p in partials]
loss_s_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_w_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss_gw_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_g_test_hists= [np.empty(dataset_test_size, dtype=np.float32) for p in partials]
loss_s_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss_w_test_hist= np.empty(dataset_test_size, dtype=np.float32)
loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
loss2_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train_hist= np.empty(dataset_train_size, dtype=np.float32)
gtvar_test_hist= np.empty(dataset_test_size, dtype=np.float32)
gtvar_train = 0.0
gtvar_test = 0.0
img_gain_test0 = 1.0
img_gain_test9 = 1.0
thr=None
thr_result = None
trains_to_update = [train_next[n_train]['more_files'] for n_train in range(len(train_next))]
for epoch in range (EPOCHS_TO_RUN):
"""
update files after each epoch, all 4.
Convert to threads after testing
"""
if (FILE_UPDATE_EPOCHS > 0) and (epoch % FILE_UPDATE_EPOCHS == 0):
if not thr is None:
if thr.is_alive():
qsf.print_time("***WAITING*** until tfrecord gets loaded", end=" ")
else:
qsf.print_time("tfrecord is ***ALREADY LOADED*** ", end=" ")
thr.join()
qsf.print_time("Done")
qsf.print_time("Inserting new data", end=" ")
for n_train in range(len(trains_to_update)):
if trains_to_update[n_train]:
qsf.add_file_to_dataset(dataset = dataset_train,
new_dataset = thr_result[n_train],
train_next = train_next[n_train])
qsf.print_time("Done")
thr_result = []
fpaths = []
for n_train in range(len(trains_to_update)):
if trains_to_update[n_train]:
fpaths.append(files['train'][n_train][train_next[n_train]['file']])
qsf.print_time("Will read in background: "+fpaths[-1])
thr = Thread(target=qsf.getMoreFiles, args=(fpaths,thr_result, CLUSTER_RADIUS, HOR_FLIP, TILE_LAYERS, TILE_SIDE))
thr.start()
train_buf_index = epoch % TRAIN_BUFFER_CPU # GPU memory from CPU memory (now 4)
if epoch >=600:
learning_rate = LR600
elif epoch >=400:
learning_rate = LR400
elif epoch >=200:
learning_rate = LR200
elif epoch >=100:
learning_rate = LR100
else:
learning_rate = LR
if (train_buf_index == 0) and SHUFFLE_FILES:
qsf.print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
qsf.shuffle_in_place(
dataset_data = dataset_train, #alternating clusters from 4 sources.each cluster has all needed data (concatenated)
period = num_train_subs)
qsf.print_time(" Done")
sti = train_buf_index * dataset_train_size * BATCH_SIZE # TRAIN_BUFFER_GPU * num_train_subs
eti = sti+ dataset_train_size * BATCH_SIZE# (train_buf_index +1) * TRAIN_BUFFER_GPU * num_train_subs
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_train[sti:eti,:corr2d_len],
target_disparity_train_placeholder: dataset_train[sti:eti,corr2d_len:corr2d_len+target_disparity_len],
gt_ds_train_placeholder: dataset_train[sti:eti,corr2d_len+target_disparity_len:corr2d_len+target_disparity_len+gtds_len] })
for i in range(dataset_train_size):
# try:
_, GW_loss_trained, G_losses_trained, S_loss_trained, W_loss_trained, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[ G_opt,
GW_loss,
G_losses,
S_loss,
W_loss,
outs[0],
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={tf_batch_weights: feed_batch_weights,
lr: learning_rate
}) # previous value of *_avg #Fetch argument 0.0 has invalid type <class 'float'>, must be a string or Tensor. (Can not convert a float into a Tensor or Operation.)
loss_gw_train_hist[i] = GW_loss_trained
for nn, gl in enumerate(G_losses_trained):
loss_g_train_hists[nn][i] = gl
loss_s_train_hist[i] = S_loss_trained
loss_w_train_hist[i] = W_loss_trained
loss2_train_hist[i] = out_cost1
gtvar_train_hist[i] = gt_variance
# except tf.errors.OutOfRangeError:
# print("****** NO MORE DATA! train done at step %d"%(i))
# break
tt_gw_avg[0] = np.average(loss_gw_train_hist).astype(np.float32)
for nn, lgth in enumerate(loss_g_train_hists):
tt_g_avgs[0][nn] = np.average(lgth).astype(np.float32)
tt_s_avg[0] = np.average(loss_s_train_hist).astype(np.float32)
tt_w_avg[0] = np.average(loss_w_train_hist).astype(np.float32)
tt2_avg[0] = np.average(loss2_train_hist).astype(np.float32)
tt_gtvar_avg[0] = np.average(gtvar_train_hist).astype(np.float32)
for ntest,dataset_test in enumerate(datasets_test):
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_test[:, :corr2d_len], #['corr2d'],
target_disparity_train_placeholder: dataset_test[:, corr2d_len:corr2d_len+target_disparity_len], # ['target_disparity'],
gt_ds_train_placeholder: dataset_test[:, corr2d_len+target_disparity_len:] }) # ['gt_ds']})
"""
TODO: Make it possible to have different length dataset_test arrays to mix different length test files
"""
for i in range(dataset_test_size):
# for i in range(dataset_test.shape[0]):
# try:
GW_loss_tested, G_losses_tested, S_loss_tested, W_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
[GW_loss,
G_losses,
S_loss,
W_loss,
outs[0],
_disp_slice,
_d_gt_slice,
_out_diff,
_out_diff2,
_w_norm,
_out_wdiff2,
_cost1,
GT_variance
],
feed_dict={tf_batch_weights: feed_batch_weight_1 , # feed_batch_weights,
lr: learning_rate
}) # previous value of *_avg
loss_gw_test_hist[i] = GW_loss_tested
for nn, gl in enumerate(G_losses_tested):
loss_g_test_hists[nn][i] = gl
loss_s_test_hist[i] = S_loss_tested
loss_w_test_hist[i] = W_loss_tested
loss2_test_hist[i] = out_cost1
gtvar_test_hist[i] = gt_variance
# except tf.errors.OutOfRangeError:
# print("test done at step %d"%(i))
# break
tt_gw_avg[ntest+1] = np.average(loss_gw_test_hist).astype(np.float32)
for nn, lgth in enumerate(loss_g_test_hists):
tt_g_avgs[ntest+1][nn] = np.average(lgth).astype(np.float32)
tt_s_avg[ntest+1] = np.average(loss_s_test_hist).astype(np.float32)
tt_w_avg[ntest+1] = np.average(loss_w_test_hist).astype(np.float32)
tt2_avg[ntest+1] = np.average(loss2_test_hist).astype(np.float32)
tt_gtvar_avg[ntest+1] = np.average(gtvar_test_hist).astype(np.float32)
if (((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0)) and (len(image_data) > 0) :
lf = None
if (epoch + 1) == EPOCHS_TO_RUN: # last
print("Last epoch, removing train/test datasets to reduce memory footprint")
del(dataset_train)
del(dataset_test)
if LOGPATH:
lf=open(LOGPATH,"w") #overwrite previous (or make it "a"?
last_epoch = (epoch + 1) == EPOCHS_TO_RUN
ind_img = [0]
if last_epoch:
ind_img = [i for i in range(len(image_data))]
###################################################
# Read the full image
###################################################
## test_summaries_img = [0.0]*len(ind_img) # datasets_img)
disp_out= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_cost_nw= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_cost_w= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_d= np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_avg_disparity = np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_gt_disparity = np.empty((WIDTH*HEIGHT), dtype=np.float32)
dbg_offs = np.empty((WIDTH*HEIGHT), dtype=np.float32)
for ntest in ind_img: # datasets_img):
dataset_img = qsf.readImageData(
image_data = image_data,
files = files,
indx = ntest,
cluster_radius = CLUSTER_RADIUS,
tile_layers = TILE_LAYERS,
tile_side = TILE_SIDE,
width = IMG_WIDTH,
replace_nans = True)
sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder: dataset_img['corr2d'],
target_disparity_train_placeholder: dataset_img['target_disparity'],
gt_ds_train_placeholder: dataset_img['gt_ds']})
for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
# try:
output, cost_nw, cost_w, dd, avg_disparity, gt_disparity, offs = sess.run(
[outs[0], # {?,1]
rslt_cost_nw, #[?,]
rslt_cost_w, #[?,]
rslt_d, #[?,]
rslt_avg_disparity,
rslt_gt_disparity,
rslt_offs
],
feed_dict={
tf_batch_weights: feed_batch_weight_1 # feed_batch_weights,
}) # previous value of *_avg
# except tf.errors.OutOfRangeError:
# print("test done at step %d"%(i))
# break
# try:
disp_out[start_offs:end_offs] = output.flatten()
dbg_cost_nw[start_offs:end_offs] = cost_nw.flatten()
dbg_cost_w [start_offs:end_offs] = cost_w.flatten()
dbg_d[start_offs:end_offs] = dd.flatten()
dbg_avg_disparity[start_offs:end_offs] = avg_disparity.flatten()
dbg_gt_disparity[start_offs:end_offs] = gt_disparity.flatten()
dbg_offs[start_offs:end_offs] = offs.flatten()
# except ValueError:
# print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
# break;
pass
result_file = files['result'][ntest] # result_files[ntest]
try:
os.makedirs(os.path.dirname(result_file))
except:
pass
rslt = np.concatenate(
[disp_out.reshape(-1,1),
dataset_img['t_disps'], #t_disps[ntest],
dataset_img['gtruths'], # gtruths[ntest],
dbg_cost_nw.reshape(-1,1),
dbg_cost_w.reshape(-1,1),
dbg_d.reshape(-1,1),
dbg_avg_disparity.reshape(-1,1),
dbg_gt_disparity.reshape(-1,1),
dbg_offs.reshape(-1,1)],1)
np.save(result_file, rslt.reshape(HEIGHT,WIDTH,-1))
rslt = qsf.eval_results(result_file, ABSOLUTE_DISPARITY, radius=CLUSTER_RADIUS, logfile=lf)
img_gain_test0 = rslt[0][0]/rslt[0][1]
img_gain_test9 = rslt[9][0]/rslt[9][1]
if SAVE_TIFFS:
qsf.result_npy_to_tiff(result_file, ABSOLUTE_DISPARITY, fix_nan = True,labels=SLICE_LABELS, logfile=lf)
"""
Remove dataset_img (if it is not [0] to reduce memory footprint
"""
if ntest > 0:
image_data[ntest] = None
if lf:
lf.close()
# tensorboard scalars
tt_summaries[0] = sess.run([merged],
feed_dict={ tf_ph_GW_loss: tt_gw_avg[0],
tf_ph_G_loss: tt_g_avgs[0][0], #train_g_avg,
tf_ph_G_losses: tt_g_avgs[0],
tf_ph_S_loss: tt_s_avg[0],
tf_ph_W_loss: tt_w_avg[0],
tf_ph_sq_diff: tt2_avg[0], # train2_avg,
tf_gtvar_diff: tt_gtvar_avg[0],
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9}) # previous value of *_avg #Fetch argument 0.0 has invalid type <class 'float'>, must be a string or Tensor. (Can not convert a float into a Tensor or Operation.)
for ntest, _ in enumerate(datasets_test):
tt_summaries[ntest+1] = sess.run([merged],
feed_dict={
tf_ph_GW_loss: tt_gw_avg[ntest+1],
tf_ph_G_loss: tt_g_avgs[ntest+1][0],
tf_ph_G_losses: tt_g_avgs[ntest+1], # train_g_avgs, # temporary, there is o data fro test
tf_ph_S_loss: tt_s_avg[ntest+1],
tf_ph_W_loss: tt_w_avg[ntest+1],
tf_ph_sq_diff: tt2_avg[ntest+1], #test2_avg,
tf_gtvar_diff: tt_gtvar_avg[ntest+1],
tf_img_test0: img_gain_test0,
tf_img_test9: img_gain_test9})
for n,tt_writer in enumerate(tt_writers):
## tt_writer.add_summary(tt_summaries[n],epoch)
tt_writer.add_summary(tt_summaries[n][0],epoch)
# if epoch ==0 :
# print ("adding delay to make directory creation time different")
# time.sleep(2.0) # reduce later
qsf.print_time("==== %04d:%03d -> %.4f %.4f %.4f %.4f %.4f (%.4f %.4f %.4f %.4f %.4f) ===="%(
epoch,i,
tt_gw_avg[0], tt_gw_avg[1], tt_gw_avg[2], tt_gw_avg[3], tt_gw_avg[4],
tt2_avg[0], tt2_avg[1], tt2_avg[2], tt2_avg[3], tt2_avg[4]))
if (not CHECKPOINT_PERIOD is None) and (((epoch + 1) % CHECKPOINT_PERIOD) == 0):
print("Saving periodic checkpoint (trained variables only) to %s, global_step = %d"%(os.path.dirname(files['checkpoints']), epoch),end=" => ")
print(saver.save(sess, files['checkpoints'], global_step=epoch, write_meta_graph=False))
# Close writers
for tt_writer in tt_writers:
try:
tt_writer.close()
except:
print ("Could not close tt_writer: ",tt_writer)
print("Saving final checkpoint (trained variables only) to %s"%(files['checkpoints']),end=" => ")
print(saver.save(sess, files["checkpoints"]))
print("All done")
exit (0)
"""
Traceback (most recent call last):
File "nn_ds_neibs30.py", line 721, in <module>
tt2_avg[0], tt2_avg[1], tt2_avg[2], tt2_avg[3], tt2_avg[4]))
ValueError: unsupported format character ' ' (0x20) at index 20
"""
\ No newline at end of file
#!/usr/bin/env python3
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
import os
import numpy as np
import tensorflow as tf
import xml.etree.ElementTree as ET
import time
import imagej_tiffwriter
TIME_LAST = 0
TIME_START = 0
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[38;5;214m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
BOLDWHITE = '\033[1;37m'
UNDERLINE = '\033[4m'
def print_time(txt="",end="\n"):
global TIME_LAST
t = time.time()
if txt:
txt +=" "
print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
TIME_LAST = t
DEFAULT_TITLES = [
['test_lvar', 'Test_flat_heuristic'],
['test_hvar', 'Test_edge_heuristic'],
['test_lvar1', 'Test_flat_random'],
['test_hvar1', 'Test_edge_random'],
['fake_lvar', 'Fake_flat_heuristic'],
['fake_hvar', 'Fake_edge_heuristic'],
['fake_lvar1', 'Fake_flat_random'],
['fake_hvar1', 'Fake_edge_random']]
def parseXmlConfig(conf_file, root_dir):
tree = ET.parse(conf_file)
root = tree.getroot()
parameters = {}
for p in root.find('parameters'):
## print ("p.tag=%s, p.text.stri[p()=%s"%(p.tag,p.text.strip()))
parameters[p.tag]=eval(p.text.strip())
# globals
dirs={}
for p in root.find('directories'):
dirs[p.tag]=eval(p.text.strip())
if not os.path.isabs(dirs[p.tag]):
dirs[p.tag] = os.path.join(root_dir, dirs[p.tag])
files={}
for p in root.find('files'):
files[p.tag]=eval(p.text.strip())
dbg_parameters = {}
for p in root.find('dbg_parameters'):
dbg_parameters[p.tag]=eval(p.text.strip())
return parameters, dirs, files, dbg_parameters
def defaultTestTitles(files):
test_titles = []
for f, n in DEFAULT_TITLES:
if f in files:
test_titles.append(n)
else:
test_titles.append(None)
return test_titles
def prepareFiles(dirs, files, suffix):
#MAX_FILES_PER_GROUP
for i, path in enumerate(files['train_lvar']):
files['train_lvar'][i]=os.path.join(dirs['train_lvar'], path)
for i, path in enumerate(files['train_hvar']):
files['train_hvar'][i]=os.path.join(dirs['train_hvar'], path)
for i, path in enumerate(files['train_lvar1']):
files['train_lvar1'][i]=os.path.join(dirs['train_lvar1'], path)
for i, path in enumerate(files['train_hvar1']):
files['train_hvar1'][i]=os.path.join(dirs['train_hvar1'], path)
for i, path in enumerate(files['test_lvar']):
files['test_lvar'][i]=os.path.join(dirs['test_lvar'], path)
for i, path in enumerate(files['test_hvar']):
files['test_hvar'][i]=os.path.join(dirs['test_hvar'], path)
if ('test_lvar1' in files) and ('test_lvar1' in dirs):
for i, path in enumerate(files['test_lvar1']):
files['test_lvar1'][i]=os.path.join(dirs['test_lvar1'], path)
if ('test_hvar1' in files) and ('test_hvar1' in dirs):
for i, path in enumerate(files['test_hvar1']):
files['test_hvar1'][i]=os.path.join(dirs['test_hvar1'], path)
if ('fake_lvar' in files) and ('fake_lvar' in dirs):
for i, path in enumerate(files['fake_lvar']):
files['fake_lvar'][i]=os.path.join(dirs['fake_lvar'], path)
if ('fake_hvar' in files) and ('fake_hvar' in dirs):
for i, path in enumerate(files['fake_hvar']):
files['fake_hvar'][i]=os.path.join(dirs['fake_hvar'], path)
if ('fake_lvar1' in files) and ('fake_lvar1' in dirs):
for i, path in enumerate(files['fake_lvar1']):
files['fake_lvar1'][i]=os.path.join(dirs['fake_lvar1'], path)
if ('fake_hvar' in files) and ('fake_hvar' in dirs):
for i, path in enumerate(files['fake_hvar1']):
files['fake_hvar1'][i]=os.path.join(dirs['fake_hvar1'], path)
result_files=[]
for i, path in enumerate(files['images']):
result_files.append(os.path.join(dirs['result'], path+"_"+suffix+'.npy'))
files['result'] = result_files
if not 'checkpoints' in files:
files['checkpoints'] = 'checkpoints'
if not 'checkpoints' in dirs:
dirs['checkpoints'] = dirs['result']
files['checkpoints'] = os.path.join(dirs['checkpoints'], files['checkpoints'])
if not 'inference' in files:
files['inference'] = 'inference'
if not 'inference' in dirs:
dirs['inference'] = dirs['result']
files['inference'] = os.path.join(dirs['inference'], files['inference'])
if not 'exportdir' in files:
files['exportdir'] = 'exportdir'
if not 'exportdir' in dirs:
dirs['exportdir'] = dirs['result']
files['exportdir'] = os.path.join(dirs['exportdir'], files['exportdir'])
if not 'figures' in dirs:
dirs['figures'] = os.path.join(dirs['result'],"figs")
files['train'] = [files['train_lvar'],files['train_hvar'], files['train_lvar1'], files['train_hvar1']]
files['test'] = [files['test_lvar'], files['test_hvar']]
if 'test_lvar1' in files:
files['test'].append(files['test_lvar1'])
if 'test_hvar1' in files:
files['test'].append(files['test_hvar1'])
# should be after result files
for i, path in enumerate(files['images']):
files['images'][i] = os.path.join(dirs['images'], path+'.tfrecords')
def readTFRewcordsEpoch(train_filename, cluster_radius):
if not '.tfrecords' in train_filename:
train_filename += '.tfrecords'
npy_dir_name = "npy"
dirname = os.path.dirname(train_filename)
npy_dir = os.path.join(dirname, npy_dir_name)
filebasename, _ = os.path.splitext(train_filename)
filebasename = os.path.basename(filebasename)
file_all = os.path.join(npy_dir,filebasename + '.npy')
if os.path.exists(file_all):
data = np.load (file_all)
else:
record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
corr2d_list=[]
target_disparity_list=[]
gt_ds_list = []
extra_list = []
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
corr2d_list.append (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
gt_ds_list.append (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
try:
extra_list.append (np.array(example.features.feature['extra'].float_list.value, dtype= np.float32))
except:
pass
corr2d= np.array(corr2d_list)
target_disparity = np.array(target_disparity_list)
gt_ds = np.array(gt_ds_list)
if len(extra_list):
extra = np.array(extra_list)
else:
extra = None
try:
os.makedirs(os.path.dirname(file_all))
except:
pass
if cluster_radius > 0:
reformat_to_clusters(
corr2d,
target_disparity,
gt_ds,
extra,
cluster_radius)
if not extra is None:
data = np.concatenate(
[corr2d, target_disparity, gt_ds, extra],
axis = 1)
else:
data = np.concatenate(
[corr2d, target_disparity, gt_ds],
axis = 1)
np.save(file_all, data)
return data
def getMoreFiles(fpaths,rslt, cluster_radius, hor_flip, tile_layers, tile_side):
for fpath in fpaths:
dataset = readTFRewcordsEpoch(fpath, cluster_radius)
if hor_flip:
if np.random.randint(2):
print_time("Performing horizontal flip", end=" ")
flip_horizontal(dataset, cluster_radius, tile_layers, tile_side)
print_time("Done")
rslt.append(dataset)
#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
def read_and_decode(filename_queue, featrures_per_tile):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'corr2d': tf.FixedLenFeature([featrures_per_tile],tf.float32), #string),
'target_disparity': tf.FixedLenFeature([1], tf.float32), #.string),
'gt_ds': tf.FixedLenFeature([2], tf.float32) #.string)
})
corr2d = features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
gt_ds = tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
in_features = tf.concat([corr2d,target_disparity],0)
corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
batch_size=1000, # 2,
capacity=30,
num_threads=2,
min_after_dequeue=10)
return corr2d_out, target_disparity_out, gt_ds_out
def add_margins(npa,radius, val = np.nan):
npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
npa_ext[0:radius,:,:] = val
npa_ext[radius + npa.shape[0]:,:,:] = val
npa_ext[:,0:radius,:] = val
npa_ext[:, radius + npa.shape[1]:,:] = val
return npa_ext
def add_neibs(npa_ext,radius):
height = npa_ext.shape[0]-2*radius
width = npa_ext.shape[1]-2*radius
side = 2 * radius + 1
# size = side * side
npa_neib = np.empty((height, width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
for dy in range (side):
for dx in range (side):
npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
return npa_neib.reshape(height, width, -1)
def extend_img_to_clusters(datasets_img,radius, width): # = 324):
# side = 2 * radius + 1
# size = side * side
if len(datasets_img) ==0:
return
num_tiles = datasets_img[0]['corr2d'].shape[0]
height = num_tiles // width
for rec in datasets_img:
if not rec is None:
rec['corr2d'] = add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
rec['gt_ds'] = add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
try:
rec['extra'] = add_neibs(add_margins(rec['extra'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
except:
pass
pass
def reformat_to_clusters_rec(datasets_data, cluster_radius):
cluster_size = (2 * cluster_radius + 1) * (2 * cluster_radius + 1)
# Reformat input data
for rec in datasets_data:
rec['corr2d'] = rec['corr2d'].reshape( (rec['corr2d'].shape[0]//cluster_size, rec['corr2d'].shape[1] * cluster_size))
rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size))
rec['gt_ds'] = rec['gt_ds'].reshape( (rec['gt_ds'].shape[0]//cluster_size, rec['gt_ds'].shape[1] * cluster_size))
try:
rec['extra'] = rec['extra'].reshape( (rec['extra'].shape[0]//cluster_size, rec['extra'].shape[1] * cluster_size))
except:
pass
def reformat_to_clusters(
corr2d,
target_disparity,
gt_ds,
extra, # may be None
cluster_radius):
cluster_size = (2 * cluster_radius + 1) * (2 * cluster_radius + 1)
# Reformat input data
corr2d.shape = ((corr2d.shape[0]//cluster_size, corr2d.shape[1] * cluster_size))
target_disparity.shape = ((target_disparity.shape[0]//cluster_size, target_disparity.shape[1] * cluster_size))
gt_ds.shape = ((gt_ds.shape[0]//cluster_size, gt_ds.shape[1] * cluster_size))
if not extra is None:
extra.shape = ((extra.shape[0]//cluster_size, extra.shape[1] * cluster_size))
def get_lengths(
cluster_radius,
tile_layers,
tile_side):
cluster_side = 2 * cluster_radius + 1
cl = cluster_side * cluster_side * tile_layers * tile_side * tile_side
tl = cluster_side * cluster_side
gl = cluster_side * cluster_side * 2 # disparity+strength, resto goes to extra
return cl, tl, gl, cluster_side
def flip_horizontal(dataset, cluster_radius, tile_layers, tile_side):
cl, tl, gl, cluster_side = get_lengths(cluster_radius, tile_layers, tile_side)
corr2d = dataset[:,:cl] .reshape([dataset.shape[0], cluster_side, cluster_side, tile_layers, tile_side, tile_side])
target_disparity = dataset[:,cl:cl+tl].reshape([dataset.shape[0], cluster_side, cluster_side, -1])
gt_ds = dataset[:,cl+tl:cl+tl+gl] .reshape([dataset.shape[0], cluster_side, cluster_side, -1])
# no extra here !
"""
Horizontal flip of tiles
"""
corr2d = corr2d[:,:,::-1,...]
target_disparity = target_disparity[:,:,::-1,...]
gt_ds = gt_ds[:,:,::-1,...]
corr2d[:,:,:,0,:,:] = corr2d[:,:,:,0,::-1,:] # flip vertical layer0 (hor)
corr2d[:,:,:,1,:,:] = corr2d[:,:,:,1,:,::-1] # flip horizontal layer1 (vert)
corr2d_2 = corr2d[:,:,:,3,::-1,:].copy() # flip vertical layer3 (diago)
corr2d[:,:,:,3,:,:] = corr2d[:,:,:,2,::-1,:] # flip vertical layer2 (diago)
corr2d[:,:,:,2,:,:] = corr2d_2
"""
pack back into a single (input)array
"""
dataset[:,:cl] = corr2d.reshape((corr2d.shape[0],-1))
dataset[:,cl:cl+tl] = target_disparity.reshape((target_disparity.shape[0],-1))
dataset[:,cl+tl:] = gt_ds.reshape((gt_ds.shape[0],-1))
def replace_nan(datasets_data): # , cluster_radius):
# Reformat input data
for rec in datasets_data:
if not rec is None:
np.nan_to_num(rec['corr2d'], copy = False)
np.nan_to_num(rec['target_disparity'], copy = False)
if 'gt_ds' in rec:
np.nan_to_num(rec['gt_ds'], copy = False)
if 'extra' in rec:
np.nan_to_num(rec['extra'], copy = False)
def permute_to_swaps(perm):
pairs = []
for i in range(len(perm)):
w = np.where(perm == i)[0][0]
if w != i:
pairs.append([i,w])
perm[w] = perm[i]
perm[i] = i
return pairs
def shuffle_in_place(dataset_data, #alternating clusters from 4 sources.each cluster has all needed data (concatenated)
period):
for i in range (period):
np.random.shuffle(dataset_data[i::period])
def add_file_to_dataset(dataset, new_dataset, train_next):
train_next['file'] = (train_next['file']+1)%train_next['files']
l = new_dataset.shape[0] * train_next['step']
if (train_next['entry'] + l) < (train_next['entries']+train_next['step']):
dataset[train_next['entry']:train_next['entry']+l:train_next['step']] = new_dataset
train_next['entry'] += l
if (train_next['entry'] >= train_next['entries']):
train_next['entry'] -= train_next['entries']
return True
else:
return False
else: # split it two parts
l = (train_next['entries'] - train_next['entry'] + (train_next['step']-1)) // train_next['step']
dataset[train_next['entry']::train_next['step']] = new_dataset[:l]
train_next['entry'] = (train_next['entry'] + l * train_next['step']) % train_next['entries'] #0,1,2,3
l1 = new_dataset.shape[0] - l # remainder
ln = train_next['entry'] + l1 * train_next['step']
dataset[train_next['entry']:ln:train_next['step']] = new_dataset[l:]
train_next['entry'] = ln
return True
"""
train_next[n_train]
Read as many files as needed, possibly repeating, until each buffer is f
"""
def initTrainTestData(
files,
cluster_radius,
buffer_size, # number of clusters per train
test_titles = None
):
"""
Generates a single np array for training with concatenated cluster of corr2d,
cluster of target_disparity, and cluster of gt_ds for convenient shuffling
"""
num_trains = len(files['train'])
num_entries = num_trains * buffer_size
dataset_train_merged = None
train_next = [None]*num_trains
for n_train, f_train in enumerate(files['train']):
train_next[n_train] = {'file':0, 'entry':n_train, 'files':len(f_train), 'entries': num_entries, 'step':num_trains, 'more_files':False}
buffer_full = False
while not buffer_full:
for fpath in f_train:
print_time("Importing train data "+(["low variance","high variance", "low variance1","high variance1"][n_train]) +" from "+fpath, end="")
new_dataset = readTFRewcordsEpoch(fpath, cluster_radius)
if dataset_train_merged is None:
dataset_train_merged = np.empty([num_entries,new_dataset.shape[1]], dtype =new_dataset.dtype)
## print("\nbefore add_file_to_dataset: train_next["+str(n_train)+"]=",train_next[n_train])
rollover = add_file_to_dataset(
dataset = dataset_train_merged,
new_dataset = new_dataset,
train_next = train_next[n_train])
## print("after add_file_to_dataset: train_next["+str(n_train)+"]=",train_next[n_train])
print_time(" Done")
if rollover:
buffer_full = True
train_next[n_train][ 'more_files'] = train_next[n_train][ 'file'] < train_next[n_train][ 'files'] # Not all files used, need to load during training
break
if test_titles is None:
test_titles = defaultTestTitles(files)
datasets_test = []
for t,v in zip(test_titles,DEFAULT_TITLES):
if not t is None:
grp = v[0]
for fpath in files[grp]:
print_time("Importing test data ("+grp+") from "+fpath, end="")
new_dataset = readTFRewcordsEpoch(fpath, cluster_radius)
datasets_test.append(new_dataset)
print_time(" Done")
"""
for grp in ['test_lvar','test_hvar','test_lvar1','test_hvar1']:
if grp in files:
for fpath in files[grp]:
print_time("Importing test data ("+grp+") from "+fpath, end="")
new_dataset = readTFRewcordsEpoch(fpath, cluster_radius)
datasets_test.append(new_dataset)
print_time(" Done")
"""
return train_next, dataset_train_merged, datasets_test
def get_full_tile_indices2d(height,width):
a = np.empty([height,width,2], dtype=np.int32)
a[...,0] = np.arange(height).reshape([height,1])
a[...,1] = np.arange(width)
return a.reshape(-1,2)
def get_full_tile_indices(height,width):
return np.arange(height*width).reshape(-1,1)
def readImageData(image_data,
files,
indx,
cluster_radius,
tile_layers,
tile_side,
width,
replace_nans,
infer = False,
keep_gt = False):
cl, tl, gl, _ = get_lengths(0, tile_layers, tile_side)
if image_data[indx] is None:
dataset = readTFRewcordsEpoch(
files['images'][indx],
cluster_radius = 0)
corr2d = dataset[:,:cl]
target_disparity = dataset[:,cl:cl+tl]
if infer:
image_data[indx] = {
'corr2d': corr2d,
'target_disparity': target_disparity,
'xy': get_full_tile_indices2d(corr2d.shape[0]//width, width),
'ntile': get_full_tile_indices(corr2d.shape[0]//width, width)}
if keep_gt:
gt_ds = dataset[:,cl+tl:cl+tl+gl]
image_data[indx]["gt_ds"] = gt_ds
image_data[indx]["gtruths"]= gt_ds.copy()
image_data[indx]["t_disps"]= target_disparity.reshape([-1,1]).copy()
else:
gt_ds = dataset[:,cl+tl:cl+tl+gl]
extra = dataset[:,cl+tl+gl:]
image_data[indx] = {
'corr2d': corr2d,
'target_disparity': target_disparity,
"gt_ds": gt_ds,
"gtruths": gt_ds.copy(), # never used?
"t_disps": target_disparity.reshape([-1,1]).copy(),
"extra": extra, # will be increased by 25
"t_extra": extra.copy() } #will still be (ntiles,3)
if cluster_radius > 0:
extend_img_to_clusters(
[image_data[indx]],
cluster_radius,
width)
if replace_nans:
replace_nan([image_data[indx]])
return image_data[indx]
def initImageData(files,
max_imgs,
cluster_radius,
tile_layers,
tile_side,
width,
replace_nans,
infer = False,
keep_gt = False):
# no_train = False):
num_imgs = len(files['images'])
img_data = [None] * num_imgs
for nfile in range(min(num_imgs, max_imgs)):
print_time("Importing test image data from "+ files['images'][nfile], end="")
readImageData(img_data,
files,
nfile,
cluster_radius,
tile_layers,
tile_side,
width,
replace_nans,
infer = infer,
keep_gt = keep_gt)
print_time(" Done")
return img_data
def evaluateAllResults(result_files, absolute_disparity, cluster_radius, labels=None, logpath=None):
if logpath:
lf=open(logpath,"w")
else:
lf = None
for result_file in result_files:
try:
print_time("Reading resuts from "+result_file, end=" ")
eval_results(result_file, absolute_disparity, radius=cluster_radius, logfile=lf)
except:
print_time(" - does not exist")
continue
print_time("Done")
print_time("Saving resuts to tiff", end=" ")
result_npy_to_tiff(result_file, absolute_disparity, fix_nan = True, labels=labels)
print_time("Done")
if lf:
lf.close()
def result_npy_prepare(npy_path, absolute, fix_nan, insert_deltas=True,labels=None):
"""
@param npy_path full path to the npy file with 4-layer data (242,324,4) - nn_disparity(offset), target_disparity, gt disparity, gt strength
data will be written as 4-layer tiff, extension '.npy' replaced with '.tiff'
@param absolute - True - the first layer contains absolute disparity, False - difference from target_disparity
@param fix_nan - replace nan in target_disparity with 0 to apply offset, target_disparity will still contain nan
@parame insert_deltas: +1 - add delta layers, +2 - add variance (max - min of this and 8 neighbors)
"""
data = np.load(npy_path) #(324,242,4) [nn_disp, target_disp,gt_disp, gt_conf]
if labels is None:
labels = ["chn%d"%(i) for i in range(data.shape[2])]
# labels = ["nn_out","hier_out","gt_disparity","gt_strength"]
nn_out = 0
# target_disparity = 1
gt_disparity = 2
gt_strength = 3
heur_err = 7
min_heur_err = 0.001
height = data.shape[0]
width = data.shape[1]
nocenter9 = np.array([[[1,1,1,1,np.nan,1,1,1,1]]], dtype = data.dtype)
if not absolute:
if fix_nan:
data[...,nn_out] += np.nan_to_num(data[...,1], copy=True)
else:
data[...,nn_out] += data[...,1]
if (insert_deltas & 1):
np.nan_to_num(data[...,gt_strength], copy=False)
data = np.concatenate([data[...,0:4],data[...,0:2],data[...,0:2],data[...,4:]], axis = 2) # data[...,4:] may be empty
labels = labels[:4]+["nn_out","hier_out","nn_err","hier_err"]+labels[4:]
data[...,6] -= data[...,gt_disparity]
data[...,7] -= data[...,gt_disparity]
for l in [2, 4, 5, 6, 7]:
if l < data.shape[2]:
data[...,l] = np.select([data[...,gt_strength]==0.0, data[...,gt_strength]>0.0], [np.nan,data[...,l]])
# All other layers - mast too
for l in range(8,data.shape[2]):
data[...,l] = np.select([data[...,gt_strength]==0.0, data[...,gt_strength]>0.0], [np.nan,data[...,l]])
"""
Calculate bad tiles where ggt was used as a master, to remove them from the results (later versions add random error)
"""
bad1 = abs(data[...,heur_err]) < min_heur_err
bad1_ext = np.concatenate([bad1 [0:1,:], bad1 [0:1,:], bad1[:,:], bad1 [-1:height,:], bad1 [-1:height,:]],axis = 0)
bad1_ext = np.concatenate([bad1_ext[:,0:1], bad1_ext[:,0:1], bad1_ext[:,:], bad1_ext[:,-1:width], bad1_ext[:,-1:width]], axis = 1)
bad25 = np.empty(shape=[height, width, 25], dtype=bad1.dtype)
bm25=np.array([[[1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1]]])
bm09=np.array([[[0,0,0,0,0, 0,1,1,1,0, 0,1,1,1,0, 0,1,1,1,0, 0,0,0,0,0]]])
bm01=np.array([[[0,0,0,0,0, 0,0,0,0,0, 0,0,1,0,0, 0,0,0,0,0, 0,0,0,0,0]]])
for row in range(5):
for col in range(5):
pass
bad25 [...,row*5+col]= bad1_ext[row:height+row, col:width+col]
bad_num1=(np.sum(bad25*bm25,axis=2) > 0).astype(data.dtype)
bad_num2=(np.sum(bad25*bm09,axis=2) > 0).astype(data.dtype)
bad_num3=(np.sum(bad25*bm01,axis=2) > 0).astype(data.dtype)
bad_num = bad_num1 + bad_num2 + bad_num3
if (insert_deltas & 2):
wo = 0.7 # ortho
wc = 0.5 #corner
w8=np.array([wc,wo,wc,wo,0.0,wo,wc,wo,wc], dtype=data.dtype)
w8/=np.sum(w8) #normalize
gt_ext = np.concatenate([data[0:1,:,gt_disparity],data[:,:,gt_disparity],data[-1:height,:,gt_disparity]],axis = 0)
gt_ext = np.concatenate([gt_ext[:,0:1], gt_ext[:,:], gt_ext[:,-1:width]],axis = 1)
gs_ext = np.concatenate([data[0:1,:,gt_strength], data[:,:,gt_strength], data[-1:height,:,gt_strength]],axis = 0)
gs_ext = np.concatenate([gs_ext[:,0:1], gs_ext[:,:], gs_ext[:,-1:width]],axis = 1)
data9 = np.empty(shape=[height, width, 9], dtype=data.dtype)
weight9 = np.empty(shape=[height, width, 9], dtype=data.dtype)
for row in range(3):
for col in range(3):
pass
data9 [...,row*3+col]= gt_ext[row:height+row, col:width+col]
weight9[...,row*3+col]= gs_ext[row:height+row, col:width+col]
data9 *= weight9/weight9 # make data=nan where wigth is 0
# data = np.concatenate([data[...],np.empty_like(data[...,-1])], axis = 2) # data[...,4:] may be empty
data = np.concatenate([data[...],np.empty(shape=[height,width,4],dtype=data.dtype)], axis = 2) # data[...,4:] may be empty
data[...,-4] = np.nanmax(data9*nocenter9, axis=2)-np.nanmin(data9*nocenter9,axis=2)# will ignore nan
np.nan_to_num(data9,copy=False) # replace all nan in data9 with 0.
weight9 *= w8
w_center = np.sum(weight9, axis=2)
dw_center = np.sum(data9*weight9, axis=2)
dw_center /= w_center # now dw_center - weighted average in the center
data[...,-3] = np.abs(data[...,gt_disparity]- dw_center)
# data[...,-2] = data[...,gt_disparity]- dw_center
#data[...,-3] *= (data[...,-4] < 1.0) # just temporary
#data[...,-3] *= (data[...,gt_disparity] < 5) #just temporary
data[...,-2] =bad_num.astype(data.dtype)
data [...,-1]= np.sum(np.nan_to_num(weight9/weight9),axis=2).astype(data.dtype)
# data[...,-1] = dw_center
labels +=["max-min","abs-center","badness","neibs"]
#neib = np.concatenate([gt_ext[:height,:width,:],],axis = )
pass
return data, labels
def result_npy_to_tiff(npy_path,
absolute,
fix_nan,
insert_deltas=True,
labels = None,
logfile = None):
"""
@param npy_path full path to the npy file with 4-layer data (242,324,4) - nn_disparity(offset), target_disparity, gt disparity, gt strength
data will be written as 4-layer tiff, extension '.npy' replaced with '.tiff'
@param absolute - True - the first layer contains absolute disparity, False - difference from target_disparity
@param fix_nan - replace nan in target_disparity with 0 to apply offset, target_disparity will still contain nan
"""
data,labels = result_npy_prepare(npy_path, absolute, fix_nan, insert_deltas, labels=labels)
tiff_path = npy_path.replace('.npy','.tiff')
data = data.transpose(2,0,1)
print("Saving results to TIFF: "+tiff_path)
if (logfile):
print("Saving results to TIFF: "+tiff_path,file=logfile)
imagej_tiffwriter.save(tiff_path,data,labels=labels)
def eval_results(rslt_path, absolute,
min_disp = -0.1, #minimal GT disparity
max_disp = 20.0, # maximal GT disparity
max_ofst_target = 1.0,
max_ofst_result = 1.0,
str_pow = 2.0,
radius = 0,
logfile = None):
variants = [[ -0.1, 5.0, 0.5, 0.5, 1.0],
[ -0.1, 5.0, 0.5, 0.5, 2.0],
[ -0.1, 5.0, 0.2, 0.2, 1.0],
[ -0.1, 5.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 0.5, 0.5, 1.0],
[ -0.1, 20.0, 0.5, 0.5, 2.0],
[ -0.1, 20.0, 0.2, 0.2, 1.0],
[ -0.1, 20.0, 0.2, 0.2, 2.0],
[ -0.1, 20.0, 1.0, 1.0, 1.0],
[min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
rslt = np.load(rslt_path)
not_nan = ~np.isnan(rslt[...,0])
not_nan &= ~np.isnan(rslt[...,1])
not_nan &= ~np.isnan(rslt[...,2])
not_nan &= ~np.isnan(rslt[...,3])
not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool)
not_nan_ext[radius:-radius,radius:-radius] = not_nan
for dy in range(2*radius+1):
for dx in range(2*radius+1):
not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
not_nan = not_nan_ext[radius:-radius,radius:-radius]
if not absolute:
rslt[...,0] += rslt[...,1]
nn_disparity = np.nan_to_num(rslt[...,0], copy = False)
target_disparity = np.nan_to_num(rslt[...,1], copy = False)
gt_disparity = np.nan_to_num(rslt[...,2], copy = False)
gt_strength = np.nan_to_num(rslt[...,3], copy = False)
rslt = []
print ("--------------- %s ---------------"%(rslt_path))
if logfile:
print ("--------------- %s ---------------"%(rslt_path), file=logfile)
for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
good_tiles = not_nan.copy();
good_tiles &= (gt_disparity >= min_disparity)
good_tiles &= (gt_disparity <= max_disparity)
good_tiles &= (target_disparity != gt_disparity)
good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
gt_w = gt_strength * good_tiles
gt_w = np.power(gt_w,strength_pow)
sw = gt_w.sum()
diff0 = target_disparity - gt_disparity
diff1 = nn_disparity - gt_disparity
diff0_2w = gt_w*diff0*diff0
diff1_2w = gt_w*diff1*diff1
rms0 = np.sqrt(diff0_2w.sum()/sw)
rms1 = np.sqrt(diff1_2w.sum()/sw)
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
if logfile:
print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ),file=logfile)
rslt.append([rms0,rms1])
return rslt
def concentricSquares(radius):
side = 2 * radius + 1
return [[((i // side) >= var) and
((i // side) < (side - var)) and
((i % side) >= var) and
((i % side) < (side - var)) for i in range (side*side) ] for var in range(radius+1)]
#!/usr/bin/env python3
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
#from numpy import float64
#import numpy as np
import tensorflow as tf
def smoothLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch_clust, # [batch_size,25,2] tf placeholder
clip, # limit punishment for cutting corners (disparity pix)
absolute_disparity = False, #when false there should be no activation on disparity output !
cluster_radius = 2):
with tf.name_scope("SmoothLoss"):
center_tile_index = 2 * cluster_radius * (cluster_radius + 1)
cluster_side = 2 * cluster_radius + 1
cluster_size = cluster_side * cluster_side
w_corner = 0.7
w8 = [w_corner,1.0,w_corner,1.0,1.0,w_corner,1.0,w_corner]
w8 = [w/sum(w8) for w in w8]
tf_w8=tf.reshape(tf.constant(w8, dtype=tf.float32, name="w8_"), shape=[1,-1], name="w8")
i8 = []
for dy in [-1,0,1]:
for dx in [-1,0,1]:
if (dy != 0) or (dx != 0):
i8.append(center_tile_index+(dy*cluster_side)+dx)
tf_clip = tf.constant(clip, dtype=tf.float32, name = "clip")
tf_gt_ds_all = tf.reshape(gt_ds_batch_clust,[-1,cluster_size,gt_ds_batch_clust.shape[1]//cluster_size], name = "gt_ds_all")
tf_neibs8 = tf.gather(tf_gt_ds_all, indices = i8, axis = 1, name = "neibs8")
tf_gt_disparity8 = tf.reshape(tf_neibs8[:,:,0], [-1,8], name = "gt8_disparity") # (?,8)
tf_gt_strength8 = tf.reshape(tf_neibs8[:,:,1], [-1,8], name = "gt8_strength") # (?,8)
tf_w = tf.multiply(tf_gt_strength8, tf_w8, name = "w")
tf_dw = tf.multiply(tf_gt_disparity8, tf_w, name = "dw")
tf_sum_w = tf.reduce_sum(tf_w, axis = 1, name = "sum_w")
tf_sum_dw = tf.reduce_sum(tf_dw, axis = 1, name = "sum_dw")
tf_avg_disparity = tf.divide(tf_sum_dw, tf_sum_w, name = "avg_disparity") # (?,)
tf_gt_disparity = tf.reshape(tf_gt_ds_all[:,center_tile_index,0], [-1], name = "gt_disparity") # (?,)
"""
It is good to limit tf_gt_disparityby min/max (+margin) tf.reduce_min(tf_gt_disparity8, axis=1,...) but there could be zeros caused by undefined GT for the tile
"""
tf_gt_strength = tf.reshape(tf_gt_ds_all[:,center_tile_index,1], [-1], name = "gt_strength") # (?,)
tf_d0 = tf.abs(tf_gt_disparity - tf_avg_disparity, name = "tf_d0")
tf_d = tf.maximum(tf_d0, 0.001, name = "tf_d")
## tf_d2 = tf.multiply(tf_d, tf_d, name = "tf_d2")
tf_out = tf.reshape(out_batch[:,0],[-1], name = "tf_out")
if absolute_disparity:
tf_out_disparity = tf_out
else:
tf_out_disparity = tf.add(tf_out, tf.reshape(target_disparity_batch,[-1]),name = "out_disparity")
tf_offs = tf.subtract(tf_out_disparity, tf_avg_disparity, name = "offs")
tf_offs2 = tf.multiply(tf_offs, tf_offs, name = "offs2")
tf_offs2_d = tf.divide(tf_offs2, tf_d, name = "offs2_d")
tf_cost0 = tf.maximum(tf_d - tf_offs2_d, 0.0, name = "cost0")
tf_cost_nw = tf.minimum(tf_cost0, tf_clip, name = "cost_nw")
## tf_cost_nw = tf.maximum(tf_d2 - tf_offs2, 0.0, name = "cost_nw")
tf_cost_w = tf.multiply(tf_cost_nw, tf_gt_strength, name = "cost_w")
tf_sum_wc = tf.reduce_sum(tf_gt_strength, name = "sum_wc")
tf_sum_costw = tf.reduce_sum(tf_cost_w, name = "sum_costw")
tf_cost = tf.divide(tf_sum_costw, tf_sum_wc, name = "cost")
return tf_cost, tf_cost_nw, tf_cost_w, tf_d , tf_avg_disparity, tf_gt_disparity, tf_offs
def batchLoss(out_batch, # [batch_size,(1..2)] tf_result
target_disparity_batch, # [batch_size] tf placeholder
gt_ds_batch, # [batch_size,2] tf placeholder
batch_weights, # [batch_size] now batch index % 4 - different sources, even - low variance, odd - high variance
disp_diff_cap = 10.0, # cap disparity difference to this value (give up on large errors)
disp_diff_slope= 0.0, #allow squared error to grow above disp_diff_cap
absolute_disparity = False, #when false there should be no activation on disparity output !
use_confidence = False,
lambda_conf_avg = 0.01,
## lambda_conf_pwr = 0.1,
conf_pwr = 2.0,
gt_conf_offset = 0.08,
gt_conf_pwr = 1.0,
error2_offset = 0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
disp_wmin = 1.0, # minimal disparity to apply weight boosting for small disparities
disp_wmax = 8.0, # maximal disparity to apply weight boosting for small disparities
use_out = False): # use calculated disparity for disparity weight boosting (False - use target disparity)
with tf.name_scope("BatchLoss"):
"""
Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
"""
tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
## tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
## tf_conf_pwr = tf.constant(conf_pwr, dtype=tf.float32, name="tf_conf_pwr")
tf_gt_conf_offset = tf.constant(gt_conf_offset, dtype=tf.float32, name="tf_gt_conf_offset")
tf_gt_conf_pwr = tf.constant(gt_conf_pwr, dtype=tf.float32, name="tf_gt_conf_pwr")
tf_num_tiles = tf.shape(gt_ds_batch)[0]
tf_0f = tf.constant(0.0, dtype=tf.float32, name="tf_0f")
tf_1f = tf.constant(1.0, dtype=tf.float32, name="tf_1f")
## tf_maxw = tf.constant(1.0, dtype=tf.float32, name="tf_maxw")
tf_disp_diff_cap2= tf.constant(disp_diff_cap*disp_diff_cap, dtype=tf.float32, name="disp_diff_cap2")
tf_disp_diff_slope= tf.constant(disp_diff_slope, dtype=tf.float32, name="disp_diff_slope")
if gt_conf_pwr == 0:
w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
else:
w_slice = tf.reshape(gt_ds_batch[:,1],[-1], name = "w_gt_slice")
w_sub = tf.subtract (w_slice, tf_gt_conf_offset, name = "w_sub")
w_clip = tf.maximum(w_sub, tf_0f, name = "w_clip")
if gt_conf_pwr == 1.0:
w = w_clip
else:
w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
if use_confidence:
tf_num_tilesf = tf.cast(tf_num_tiles, dtype=tf.float32, name="tf_num_tilesf")
conf_slice = tf.reshape(out_batch[:,1],[-1], name = "conf_slice")
conf_sum = tf.reduce_sum(conf_slice, name = "conf_sum")
conf_avg = tf.divide(conf_sum, tf_num_tilesf, name = "conf_avg")
conf_avg1 = tf.subtract(conf_avg, tf_1f, name = "conf_avg1")
conf_avg2 = tf.square(conf_avg1, name = "conf_avg2")
cost2 = tf.multiply (conf_avg2, tf_lambda_conf_avg, name = "cost2")
iconf_avg = tf.divide(tf_1f, conf_avg, name = "iconf_avg")
nconf = tf.multiply (conf_slice, iconf_avg, name = "nconf") #normalized confidence
nconf_pwr = tf.pow(nconf, conf_pwr, name = "nconf_pwr")
nconf_pwr_sum = tf.reduce_sum(nconf_pwr, name = "nconf_pwr_sum")
nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f, name = "nconf_pwr_offs")
cost3 = tf.multiply (conf_avg2, nconf_pwr_offs, name = "cost3")
w_all = tf.multiply (w, nconf, name = "w_all")
else:
w_all = w
# cost2 = 0.0
# cost3 = 0.0
# normalize weights
w_sum = tf.reduce_sum(w_all, name = "w_sum")
iw_sum = tf.divide(tf_1f, w_sum, name = "iw_sum")
w_norm = tf.multiply (w_all, iw_sum, name = "w_norm")
disp_slice = tf.reshape(out_batch[:,0],[-1], name = "disp_slice")
d_gt_slice = tf.reshape(gt_ds_batch[:,0],[-1], name = "d_gt_slice")
td_flat = tf.reshape(target_disparity_batch,[-1], name = "td_flat")
if absolute_disparity:
adisp = disp_slice
else:
adisp = tf.add(disp_slice, td_flat, name = "adisp")
out_diff = tf.subtract(adisp, d_gt_slice, name = "out_diff")
out_diff2 = tf.square(out_diff, name = "out_diff2")
pre_cap0 = tf.abs(out_diff, name = "pre_cap0")
pre_cap = tf.multiply(pre_cap0, tf_disp_diff_slope, name = "pre_cap")
diff_cap = tf.add(pre_cap, tf_disp_diff_cap2, name = "diff_cap")
out_diff2_capped = tf.minimum(out_diff2, diff_cap, name = "out_diff2_capped")
out_wdiff2 = tf.multiply (out_diff2_capped, w_norm, name = "out_wdiff2")
cost1 = tf.reduce_sum(out_wdiff2, name = "cost1")
out_diff2_offset = tf.subtract(out_diff2, error2_offset, name = "out_diff2_offset")
out_diff2_biased = tf.maximum(out_diff2_offset, 0.0, name = "out_diff2_biased")
# calculate disparity-based weight boost
if use_out:
dispw = tf.clip_by_value(adisp, disp_wmin, disp_wmax, name = "dispw")
else:
dispw = tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
dispw_boost = tf.divide(disp_wmax, dispw, name = "dispw_boost")
dispw_comp = tf.multiply (dispw_boost, w_norm, name = "dispw_comp") #HERE??
if batch_weights.shape[0] > 1:
dispw_batch = tf.multiply (dispw_comp, batch_weights, name = "dispw_batch")# apply weights for high/low variance and sources
else:
dispw_batch = tf.multiply (dispw_comp, tf_1f, name = "dispw_batch")# apply weights for high/low variance and sources
dispw_sum = tf.reduce_sum(dispw_batch, name = "dispw_sum")
idispw_sum = tf.divide(tf_1f, dispw_sum, name = "idispw_sum")
dispw_norm = tf.multiply (dispw_batch, idispw_sum, name = "dispw_norm")
out_diff2_wbiased = tf.multiply(out_diff2_biased, dispw_norm, name = "out_diff2_wbiased")
# out_diff2_wbiased = tf.multiply(out_diff2_biased, w_norm, name = "out_diff2_wbiased")
cost1b = tf.reduce_sum(out_diff2_wbiased, name = "cost1b")
if use_confidence:
cost12 = tf.add(cost1b, cost2, name = "cost12")
cost123 = tf.add(cost12, cost3, name = "cost123")
return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
else:
return cost1b, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
def weightsLoss(inp_weights,
tile_layers,
tile_side,
wborders_zero):
# [batch_size,(1..2)] tf_result
# weights_lambdas): # single lambda or same length as inp_weights.shape[1]
"""
Enforcing 'smooth' weights for the input 2d correlation tiles
@return mean squared difference for each weight and average of 8 neighbors divided by mean squared weights
"""
weight_ortho = 1.0
weight_diag = 0.7
sw = 4.0 * (weight_ortho + weight_diag)
weight_ortho /= sw
weight_diag /= sw
# w_neib = tf.const([[weight_diag, weight_ortho, weight_diag],
# [weight_ortho, -1.0, weight_ortho],
# [weight_diag, weight_ortho, weight_diag]])
#WBORDERS_ZERO
with tf.name_scope("WeightsLoss"):
# Adding 1 tile border
# tf_inp = tf.reshape(inp_weights[:TILE_LAYERS * TILE_SIZE,:], [TILE_LAYERS, FILE_TILE_SIDE, FILE_TILE_SIDE, inp_weights.shape[1]], name = "tf_inp")
tf_inp = tf.reshape(inp_weights[:tile_layers * tile_side * tile_side,:], [tile_layers, tile_side, tile_side, inp_weights.shape[1]], name = "tf_inp")
if wborders_zero:
tf_zero_col = tf.constant(0.0, dtype=tf.float32, shape=[tf_inp.shape[0], tf_inp.shape[1], 1, tf_inp.shape[3]], name = "tf_zero_col")
tf_zero_row = tf.constant(0.0, dtype=tf.float32, shape=[tf_inp.shape[0], 1 , tf_inp.shape[2] + 2, tf_inp.shape[3]], name = "tf_zero_row")
tf_inp_ext_h = tf.concat([tf_zero_col, tf_inp, tf_zero_col ], axis = 2, name ="tf_inp_ext_h")
tf_inp_ext = tf.concat([tf_zero_row, tf_inp_ext_h, tf_zero_row ], axis = 1, name ="tf_inp_ext")
else:
tf_inp_ext_h = tf.concat([tf_inp [:, :, :1, :], tf_inp, tf_inp [:, :, -1:, :]], axis = 2, name ="tf_inp_ext_h")
tf_inp_ext = tf.concat([tf_inp_ext_h [:, :1, :, :], tf_inp_ext_h, tf_inp_ext_h[:, -1:, :, :]], axis = 1, name ="tf_inp_ext")
s_ortho = tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:,:] + tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:, :]
s_corn = tf_inp_ext[:, :-2,:-2,:] + tf_inp_ext[:, :-2, 2:,:] + tf_inp_ext[:,2:, :-2,:] + tf_inp_ext[:,2: , 2:, :]
w_diff = tf.subtract(tf_inp, s_ortho * weight_ortho + s_corn * weight_diag, name="w_diff")
w_diff2 = tf.multiply(w_diff, w_diff, name="w_diff2")
w_var = tf.reduce_mean(w_diff2, name="w_var")
w2_mean = tf.reduce_mean(inp_weights * inp_weights, name="w2_mean")
w_rel = tf.divide(w_var, w2_mean, name= "w_rel")
return w_rel # scalar, cost for weights non-smoothness in 2d
#!/usr/bin/env python3
__copyright__ = "Copyright 2018, Elphel, Inc."
__license__ = "GPL-3.0+"
__email__ = "andrey@elphel.com"
#from numpy import float64
#import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
NN_LAYOUTS = {0:[0, 0, 0, 32, 20, 16],
1:[0, 0, 0, 256, 128, 64],
2:[0, 128, 32, 32, 32, 16],
3:[0, 0, 40, 32, 20, 16],
4:[0, 0, 0, 0, 16, 16],
5:[0, 0, 64, 32, 32, 16],
6:[0, 0, 32, 16, 16, 16],
7:[0, 0, 64, 16, 16, 16],
8:[0, 0, 0, 64, 20, 16],
9:[0, 0, 256, 64, 32, 16],
10:[0, 256, 128, 64, 32, 16],
11:[0, 0, 0, 0, 64, 32],
12:[0, 0, 256, 128, 64, 32],
13:[0, 0, 0, 256, 128, 32],
}
def lrelu(x):
return tf.maximum(x*0.2,x)
# return tf.nn.relu(x)
def sym_inputs8(inp, cluster_radius = 2):
"""
get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
return same lengh, reordered
"""
tile_side = 2 * cluster_radius + 1
with tf.name_scope("sym_inputs8"):
td = inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
inp_corr = tf.reshape(inp[:,:-1],[-1,4,tile_side,tile_side], name = "inp_corr")
inp_corr_h = tf.stack([-inp_corr [:,0,:,-1::-1], inp_corr [:,1,:,-1::-1], -inp_corr [:,3,:,-1::-1], -inp_corr [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
inp_corr_v = tf.stack([ inp_corr [:,0,-1::-1,:],-inp_corr [:,1,-1::-1,:], inp_corr [:,3,-1::-1,:], inp_corr [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
inp_corr_hv = tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:], inp_corr_h[:,3,-1::-1,:], inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
inp_corr_t = tf.stack([tf.transpose(inp_corr [:,1], perm=[0,2,1]),
tf.transpose(inp_corr [:,0], perm=[0,2,1]),
tf.transpose(inp_corr [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
inp_corr_ht = tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
inp_corr_vt = tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
-tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
# return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
"""
return [tf.concat([tf.reshape(inp_corr, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
"""
cl = 4 * tile_side * tile_side
return [tf.concat([tf.reshape(inp_corr, [-1,cl]),td], axis=1,name = "out_corr"),
tf.concat([tf.reshape(inp_corr_h, [-1,cl]),td], axis=1,name = "out_corr_h"),
tf.concat([tf.reshape(inp_corr_v, [-1,cl]),td], axis=1,name = "out_corr_v"),
tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
tf.concat([tf.reshape(inp_corr_t, [-1,cl]),td], axis=1,name = "out_corr_t"),
tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
# inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
def network_sub(input_tensor,
input_global, #add to all layers (but first) if not None
layout,
reuse,
sym8 = False,
cluster_radius = 2):
# last_indx = None;
fc = []
inp_weights = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
if input_global is None:
inp = fc[-1]
else:
inp = tf.concat([fc[-1], input_global], axis = 1)
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
else:
inp = input_tensor
if sym8:
inp8 = sym_inputs8(inp, cluster_radius)
num_non_sum = num_outs % len(inp8) # if number of first layer outputs is not multiple of 8
num_sym8 = num_outs // len(inp8) # number of symmetrical groups
fc_sym = []
for j in range (len(inp8)): # ==8
reuse_this = reuse | (j > 0)
scp = 'g_fc_sub'+str(i)
fc_sym.append(slim.fully_connected(inp8[j], num_sym8, activation_fn=lrelu, scope= scp, reuse = reuse_this))
if not reuse_this:
with tf.compat.v1.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
if num_non_sum > 0:
reuse_this = reuse
scp = 'g_fc_sub'+str(i)+"r"
fc_sym.append(slim.fully_connected(inp, num_non_sum, activation_fn=lrelu, scope=scp, reuse = reuse_this))
if not reuse_this:
with tf.compat.v1.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
else:
scp = 'g_fc_sub'+str(i)
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope= scp, reuse = reuse))
if not reuse:
with tf.compat.v1.variable_scope(scp, reuse=True) : # tf.AUTO_REUSE):
inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs]))
return fc[-1], inp_weights
def network_inter(input_tensor,
input_global, #add to all layers (but first) if not None
layout,
reuse=False,
use_confidence=False):
#last_indx = None;
fc = []
for i, num_outs in enumerate (layout):
if num_outs:
if fc:
if input_global is None:
inp = fc[-1]
else:
inp = tf.concat([fc[-1], input_global], axis = 1)
else:
inp = input_tensor
fc.append(slim.fully_connected(inp, num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i), reuse = reuse))
if use_confidence:
fc_out = slim.fully_connected(fc[-1], 2, activation_fn=lrelu, scope='g_fc_inter_out', reuse = reuse)
else:
fc_out = slim.fully_connected(fc[-1], 1, activation_fn=None, scope='g_fc_inter_out', reuse = reuse)
#If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only
return fc_out
def networks_siam(input_tensor, # now [?,9,325]-> [?,25,325]
input_global, # add to all layers (but first) if not None
layout1,
layout2,
inter_convergence,
sym8 = False,
only_tile = None, # just for debugging - feed only data from the center sub-network
partials = None,
use_confidence=False,
cluster_radius = 2):
center_index = (input_tensor.shape[1] - 1) // 2
with tf.name_scope("Siam_net"):
inp_weights = []
num_legs = input_tensor.shape[1] # == 25
if partials is None:
partials = [[True] * num_legs]
inter_lists = [[] for _ in partials]
reuse = False
for i in range (num_legs):
if ((only_tile is None) or (i == only_tile)) and any([p[i] for p in partials]) :
if input_global is None:
ig = None
else:
ig =input_global[:,i,:]
ns, ns_weights = network_sub(input_tensor[:,i,:],
ig, # input_global[:,i,:],
layout= layout1,
reuse= reuse,
sym8 = sym8,
cluster_radius = cluster_radius)
for n, partial in enumerate(partials):
if partial[i]:
inter_lists[n].append(ns)
else:
inter_lists[n].append(tf.zeros_like(ns))
inp_weights += ns_weights
reuse = True
outs = []
for n, _ in enumerate(partials):
if input_global is None:
ig = None
else:
ig =input_global[:,center_index,:]
outs.append(network_inter (input_tensor = tf.concat(inter_lists[n],
axis=1,
name='inter_tensor'+str(n)),
input_global = [None, ig][inter_convergence], # optionally feed all convergence values (from each tile of a cluster)
layout = layout2,
reuse = (n > 0),
use_confidence = use_confidence))
return outs, inp_weights
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment