Commit 606a4277 authored by Oleg Dzhimiev's avatar Oleg Dzhimiev

Merge branch 'master' of git.elphel.com:Elphel/python3-imagej-tiff

parents 07c7d46a ded0d987
...@@ -74,8 +74,10 @@ def readTFRewcordsEpoch(train_filename): ...@@ -74,8 +74,10 @@ def readTFRewcordsEpoch(train_filename):
class ExploreData: class ExploreData:
PATTERN = "*-DSI_COMBO.tiff" PATTERN = "*-DSI_COMBO.tiff"
# ML_DIR = "ml" # ML_DIR = "ml"
ML_PATTERN = "*-ML_DATA-*.tiff" ML_PATTERN = "*-ML_DATA*.tiff"
"""
1527182801_296892-ML_DATARND-32B-O-FZ0.05-OFFS-0.20000_0.20000.tiff
"""
def getComboList(self, top_dir): def getComboList(self, top_dir):
# patt = "*-DSI_COMBO.tiff" # patt = "*-DSI_COMBO.tiff"
tlist = [] tlist = []
...@@ -377,18 +379,6 @@ class ExploreData: ...@@ -377,18 +379,6 @@ class ExploreData:
num_batch_tiles = np.empty((data_ds.shape[0],self.hist_to_batch.max()+1),dtype = int) num_batch_tiles = np.empty((data_ds.shape[0],self.hist_to_batch.max()+1),dtype = int)
bb = self.getBB(data_ds) bb = self.getBB(data_ds)
use_neibs = not ((disp_var is None) or (disp_neibs is None) or (min_var is None) or (max_var is None) or (min_neibs is None)) use_neibs = not ((disp_var is None) or (disp_neibs is None) or (min_var is None) or (max_var is None) or (min_neibs is None))
'''
bb = np.empty((data_ds.shape[0],data_ds.shape[1],data_ds.shape[2]),int)
for findx in range(data_ds.shape[0]):
ds = data_ds[findx]
gt = ds[...,1] > 0.0 # all true - check
db = (((ds[...,0] - self.disparity_min_clip)/disp_step).astype(int))*gt
sb = (((ds[...,1] - self.strength_min_clip)/ str_step).astype(int))*gt
np.clip(db, 0, self.disparity_bins-1, out = db)
np.clip(sb, 0, self.strength_bins-1, out = sb)
bb[findx] = (self.hist_to_batch[sb.reshape(self.num_tiles),db.reshape(self.num_tiles)]) .reshape(db.shape[0],db.shape[1]) + (gt -1)
pass
'''
list_of_file_lists=[] list_of_file_lists=[]
for findx in range(data_ds.shape[0]): for findx in range(data_ds.shape[0]):
foffs = findx * self.num_tiles foffs = findx * self.num_tiles
...@@ -596,6 +586,7 @@ class ExploreData: ...@@ -596,6 +586,7 @@ class ExploreData:
if not '.tfrecords' in tfr_filename: if not '.tfrecords' in tfr_filename:
tfr_filename += '.tfrecords' tfr_filename += '.tfrecords'
tfr_filename=tfr_filename.replace(' ','_')
if files_list is None: if files_list is None:
files_list = self.files_train files_list = self.files_train
...@@ -606,20 +597,33 @@ class ExploreData: ...@@ -606,20 +597,33 @@ class ExploreData:
#$ files_list = [self.files_train, self.files_test][test_set] #$ files_list = [self.files_train, self.files_test][test_set]
seed_list = np.arange(len(files_list)) seed_list = np.arange(len(files_list))
np.random.shuffle(seed_list) np.random.shuffle(seed_list)
cluster_size = (2 * radius + 1) * (2 * radius + 1)
for nscene, seed_index in enumerate(seed_list): for nscene, seed_index in enumerate(seed_list):
corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius) corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
#shuffles tiles in a batch #shuffles tiles in a batch
tiles_in_batch = len(target_disparity_batch) # tiles_in_batch = len(target_disparity_batch)
permut = np.random.permutation(tiles_in_batch) tiles_in_batch = corr2d_batch.shape[0]
corr2d_batch_shuffled = corr2d_batch[permut].reshape((corr2d_batch.shape[0], corr2d_batch.shape[1]*corr2d_batch.shape[2])) clusters_in_batch = tiles_in_batch // cluster_size
target_disparity_batch_shuffled = target_disparity_batch[permut].reshape((tiles_in_batch,1)) # permut = np.random.permutation(tiles_in_batch)
gt_ds_batch_shuffled = gt_ds_batch[permut] permut = np.random.permutation(clusters_in_batch)
corr2d_clusters = corr2d_batch. reshape((clusters_in_batch,-1))
target_disparity_clusters = target_disparity_batch.reshape((clusters_in_batch,-1))
gt_ds_clusters = gt_ds_batch. reshape((clusters_in_batch,-1))
# corr2d_batch_shuffled = corr2d_batch[permut].reshape((corr2d_batch.shape[0], corr2d_batch.shape[1]*corr2d_batch.shape[2]))
# target_disparity_batch_shuffled = target_disparity_batch[permut].reshape((tiles_in_batch,1))
# gt_ds_batch_shuffled = gt_ds_batch[permut]
corr2d_batch_shuffled = corr2d_clusters[permut]. reshape((tiles_in_batch, -1))
target_disparity_batch_shuffled = target_disparity_clusters[permut].reshape((tiles_in_batch, -1))
gt_ds_batch_shuffled = gt_ds_clusters[permut]. reshape((tiles_in_batch, -1))
if nscene == 0: if nscene == 0:
dtype_feature_corr2d = _dtype_feature(corr2d_batch_shuffled) dtype_feature_corr2d = _dtype_feature(corr2d_batch_shuffled)
dtype_target_disparity = _dtype_feature(target_disparity_batch_shuffled) dtype_target_disparity = _dtype_feature(target_disparity_batch_shuffled)
dtype_feature_gt_ds = _dtype_feature(gt_ds_batch_shuffled) dtype_feature_gt_ds = _dtype_feature(gt_ds_batch_shuffled)
for i in range(tiles_in_batch): for i in range(tiles_in_batch):
x = corr2d_batch_shuffled[i].astype(np.float32) x = corr2d_batch_shuffled[i].astype(np.float32)
y = target_disparity_batch_shuffled[i].astype(np.float32) y = target_disparity_batch_shuffled[i].astype(np.float32)
z = gt_ds_batch_shuffled[i].astype(np.float32) z = gt_ds_batch_shuffled[i].astype(np.float32)
...@@ -629,7 +633,7 @@ class ExploreData: ...@@ -629,7 +633,7 @@ class ExploreData:
example = tf.train.Example(features=tf.train.Features(feature=d_feature)) example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString()) writer.write(example.SerializeToString())
if (self.debug_level > 0): if (self.debug_level > 0):
print("Scene %d of %d"%(nscene, len(seed_list))) print("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))
writer.close() writer.close()
sys.stdout.flush() sys.stdout.flush()
...@@ -711,7 +715,7 @@ if __name__ == "__main__": ...@@ -711,7 +715,7 @@ if __name__ == "__main__":
try: try:
pathTFR = sys.argv[3] pathTFR = sys.argv[3]
except IndexError: except IndexError:
pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/tf" pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
try: try:
ml_subdir = sys.argv[4] ml_subdir = sys.argv[4]
...@@ -719,10 +723,10 @@ if __name__ == "__main__": ...@@ -719,10 +723,10 @@ if __name__ == "__main__":
ml_subdir = "ml" ml_subdir = "ml"
#Parameters to generate neighbors data. Set radius to 0 to generate single-tile #Parameters to generate neighbors data. Set radius to 0 to generate single-tile
RADIUS = 0 RADIUS = 1
MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9 MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
VARIANCE_THRESHOLD = 1.5 VARIANCE_THRESHOLD = 1.5
NUM_TRAIN_SETS = 6 NUM_TRAIN_SETS = 8
if RADIUS == 0: if RADIUS == 0:
BATCH_DISP_BINS = 20 BATCH_DISP_BINS = 20
...@@ -731,8 +735,8 @@ if __name__ == "__main__": ...@@ -731,8 +735,8 @@ if __name__ == "__main__":
BATCH_DISP_BINS = 8 BATCH_DISP_BINS = 8
BATCH_STR_BINS = 3 BATCH_STR_BINS = 3
train_filenameTFR = pathTFR+"-train" train_filenameTFR = pathTFR+"/train"
test_filenameTFR = pathTFR+"-test" test_filenameTFR = pathTFR+"/test"
# disp_bins = 20, # disp_bins = 20,
# str_bins=10) # str_bins=10)
...@@ -821,7 +825,7 @@ if __name__ == "__main__": ...@@ -821,7 +825,7 @@ if __name__ == "__main__":
pass pass
# ex_data.makeBatchLists(data_ds = ex_data.train_ds) # ex_data.makeBatchLists(data_ds = ex_data.train_ds)
for train_var in range (NUM_TRAIN_SETS): for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("-%03d"%(train_var,)) fpath = train_filenameTFR+("%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.* list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
...@@ -846,7 +850,7 @@ if __name__ == "__main__": ...@@ -846,7 +850,7 @@ if __name__ == "__main__":
num_le_train = num_batch_tiles_train.sum() num_le_train = num_batch_tiles_train.sum()
print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train)) print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train))
for train_var in range (NUM_TRAIN_SETS): for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("-%03d_R%d_LE%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)) fpath = train_filenameTFR+("%03d_R%d_LE%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.* list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
...@@ -860,7 +864,7 @@ if __name__ == "__main__": ...@@ -860,7 +864,7 @@ if __name__ == "__main__":
high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train) high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train)
print("Number of > %f disparity variance tiles: %d, fraction = %f (train)"%(VARIANCE_THRESHOLD, num_gt_train, high_fract_train)) print("Number of > %f disparity variance tiles: %d, fraction = %f (train)"%(VARIANCE_THRESHOLD, num_gt_train, high_fract_train))
for train_var in range (NUM_TRAIN_SETS): for train_var in range (NUM_TRAIN_SETS):
fpath = train_filenameTFR+("-%03d_R%d_GT%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)) fpath = (train_filenameTFR+("%03d_R%d_GT%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
# test # test
...@@ -874,7 +878,7 @@ if __name__ == "__main__": ...@@ -874,7 +878,7 @@ if __name__ == "__main__":
num_le_test = num_batch_tiles_test.sum() num_le_test = num_batch_tiles_test.sum()
print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test)) print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
fpath = test_filenameTFR +("-TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD)) fpath = test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.* list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
...@@ -887,7 +891,7 @@ if __name__ == "__main__": ...@@ -887,7 +891,7 @@ if __name__ == "__main__":
num_gt_test = num_batch_tiles_test.sum() num_gt_test = num_batch_tiles_test.sum()
high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test) high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test)) print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test))
fpath = test_filenameTFR +("-TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD)) fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
plt.show() plt.show()
......
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment