Skip to content

P
python3-imagej-tiff
Commit 94636179 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

Added zero border conditions for 1-st layer weights, generation of the 

test files same lengtha s train ones
parent 464cacc9
Expand all Show whitespace changes
Inline Side-by-side
Showing with 273 additions and 90 deletions
explore_data2.py
View file @ 94636179
...@@ -76,7 +76,7 @@ def writeTFRewcordsImageTiles(img_path, tfr_filename): # test_set=False): ...@@ -76,7 +76,7 @@ def writeTFRewcordsImageTiles(img_path, tfr_filename): # test_set=False):
num_tiles = 242*324 # fixme num_tiles = 242*324 # fixme
all_image_tiles = np.array(range(num_tiles)) all_image_tiles = np.array(range(num_tiles))
corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair'] corr_layers = ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
img = ijt.imagej_tiff(test_corr, corr_layers, all_image_tiles) img = ijt.imagej_tiff(img_path, corr_layers, all_image_tiles)
corr2d = img.corr2d.reshape((num_tiles,-1)) corr2d = img.corr2d.reshape((num_tiles,-1))
target_disparity = img.target_disparity.reshape((num_tiles,-1)) target_disparity = img.target_disparity.reshape((num_tiles,-1))
...@@ -493,7 +493,7 @@ class ExploreData: ...@@ -493,7 +493,7 @@ class ExploreData:
if nchoices >= min_choices: # use minimum of extra files if nchoices >= min_choices: # use minimum of extra files
break; break;
while len(tl)==0: while len(tl)==0:
print("** BUG! could not find a single candidate from files ",flist," for cell "+nt) print("** BUG! could not find a single candidate from files ",flist," for cell ",nt)
print("trying to use some other cell") print("trying to use some other cell")
nt1 = np.random.randint(0,self.num_batch_tiles.shape[1]) nt1 = np.random.randint(0,self.num_batch_tiles.shape[1])
for findx in flist: for findx in flist:
...@@ -753,7 +753,7 @@ ValueError: need at least one array to concatenate ...@@ -753,7 +753,7 @@ ValueError: need at least one array to concatenate
def writeTFRewcordsEpoch(self, tfr_filename, ml_list, files_list = None, set_ds= None, radius = 0): # test_set=False): def writeTFRewcordsEpoch(self, tfr_filename, ml_list, files_list = None, set_ds= None, radius = 0, num_scenes = None): # test_set=False):
# train_filename = 'train.tfrecords' # address to save the TFRecords file # train_filename = 'train.tfrecords' # address to save the TFRecords file
# open the TFRecords file # open the TFRecords file
if not '.tfrecords' in tfr_filename: if not '.tfrecords' in tfr_filename:
...@@ -777,7 +777,10 @@ ValueError: need at least one array to concatenate ...@@ -777,7 +777,10 @@ ValueError: need at least one array to concatenate
return return
writer = tf.python_io.TFRecordWriter(tfr_filename) writer = tf.python_io.TFRecordWriter(tfr_filename)
#$ files_list = [self.files_train, self.files_test][test_set] #$ files_list = [self.files_train, self.files_test][test_set]
seed_list = np.arange(len(files_list)) if num_scenes is None:
num_scenes = len(files_list)
seed_list = np.arange(num_scenes) % len(files_list)
# seed_list = np.arange(len(files_list))
np.random.shuffle(seed_list) np.random.shuffle(seed_list)
cluster_size = (2 * radius + 1) * (2 * radius + 1) cluster_size = (2 * radius + 1) * (2 * radius + 1)
for nscene, seed_index in enumerate(seed_list): for nscene, seed_index in enumerate(seed_list):
...@@ -815,7 +818,7 @@ ValueError: need at least one array to concatenate ...@@ -815,7 +818,7 @@ ValueError: need at least one array to concatenate
example = tf.train.Example(features=tf.train.Features(feature=d_feature)) example = tf.train.Example(features=tf.train.Features(feature=d_feature))
writer.write(example.SerializeToString()) writer.write(example.SerializeToString())
if (self.debug_level > 0): if (self.debug_level > 0):
print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename)) print_time("Scene %d (%d) of %d -> %s"%(nscene, seed_index, len(seed_list), tfr_filename))
writer.close() writer.close()
sys.stdout.flush() sys.stdout.flush()
...@@ -904,7 +907,7 @@ if __name__ == "__main__": ...@@ -904,7 +907,7 @@ if __name__ == "__main__":
except IndexError: except IndexError:
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/" # pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/" # pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_4" #no trailing "/" pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_5" #no trailing "/"
try: try:
ml_subdir = sys.argv[4] ml_subdir = sys.argv[4]
...@@ -914,14 +917,20 @@ if __name__ == "__main__": ...@@ -914,14 +917,20 @@ if __name__ == "__main__":
# pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/" # pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
# test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' # test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' # overlook
test_corr = '/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527256816_150165/v02/mlr32_18a/1527256816_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' # test_corr = '/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527256816_150165/v02/mlr32_18a/1527256816_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' # State Street
# test_corr = '/home/eyesis/x3d_data/models/dsi_combo_and_ml_all/state_street/1527256858_150165/v01/mlr32_18a/1527256858_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' # test_corr = '/home/eyesis/x3d_data/models/dsi_combo_and_ml_all/state_street/1527256858_150165/v01/mlr32_18a/1527256858_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff' # State Street
test_corrs = ['/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527182802_096892/v02/mlr32_18a/1527182802_096892-ML_DATA-32B-O-FZ0.05-MAIN.tiff', # near plane"
'/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527182805_096892/v02/mlr32_18a/1527182805_096892-ML_DATA-32B-O-FZ0.05-MAIN.tiff', # medium plane"
'/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527182810_096892/v02/mlr32_18a/1527182810_096892-ML_DATA-32B-O-FZ0.05-MAIN.tiff', # far plane
]
#Parameters to generate neighbors data. Set radius to 0 to generate single-tile #Parameters to generate neighbors data. Set radius to 0 to generate single-tile
TEST_SAME_LENGTH_AS_TRAIN = True # make test to have same number of entries as train ones
RADIUS = 2 # 5x5 RADIUS = 2 # 5x5
MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9 MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
VARIANCE_THRESHOLD = 1.5 VARIANCE_THRESHOLD = 1.5
NUM_TRAIN_SETS = 24 # 8 NUM_TRAIN_SETS = 32 # 8
if RADIUS == 0: if RADIUS == 0:
BATCH_DISP_BINS = 50 # 1000 * 1 BATCH_DISP_BINS = 50 # 1000 * 1
...@@ -980,6 +989,16 @@ if __name__ == "__main__": ...@@ -980,6 +989,16 @@ if __name__ == "__main__":
# RADIUS = 1 # RADIUS = 1
# MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid # MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid
# VARIANCE_THRESHOLD = 1.5 # VARIANCE_THRESHOLD = 1.5
for test_corr in test_corrs:
scene = os.path.basename(test_corr)[:17]
scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
fname =scene+'-'+scene_version
img_filenameTFR = os.path.join(pathTFR,'img',fname)
print_time("Saving test image %s as tiles..."%(img_filenameTFR),end = " ")
writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
print_time("Done")
pass
if (RADIUS > 0): if (RADIUS > 0):
disp_var_test, num_neibs_test = ex_data.exploreNeibs(ex_data.test_ds, RADIUS) disp_var_test, num_neibs_test = ex_data.exploreNeibs(ex_data.test_ds, RADIUS)
...@@ -1011,6 +1030,8 @@ if __name__ == "__main__": ...@@ -1011,6 +1030,8 @@ if __name__ == "__main__":
ml_list_train=ex_data.getMLList(ml_subdir, ex_data.files_train) ml_list_train=ex_data.getMLList(ml_subdir, ex_data.files_train)
ml_list_test= ex_data.getMLList(ml_subdir, ex_data.files_test) ml_list_test= ex_data.getMLList(ml_subdir, ex_data.files_test)
num_test_scenes = [ex_data.files_test, ex_data.files_train][TEST_SAME_LENGTH_AS_TRAIN]
if RADIUS == 0 : if RADIUS == 0 :
list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.* list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
...@@ -1034,7 +1055,7 @@ if __name__ == "__main__": ...@@ -1034,7 +1055,7 @@ if __name__ == "__main__":
max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include max_var = VARIANCE_THRESHOLD, # Maximal tile variance to include
min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include min_neibs = MIN_NEIBS) # Minimal number of valid tiles to include
fpath = test_filenameTFR # +("-%03d"%(train_var,)) fpath = test_filenameTFR # +("-%03d"%(train_var,))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_test, set_ds= ex_data.test_ds) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_test, set_ds= ex_data.test_ds, num_scenes = num_test_scenes)
pass pass
else: # RADIUS > 0 else: # RADIUS > 0
# train # train
...@@ -1079,7 +1100,7 @@ if __name__ == "__main__": ...@@ -1079,7 +1100,7 @@ if __name__ == "__main__":
print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test)) print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
fpath = test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD)) fpath = test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS, num_scenes = num_test_scenes)
list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.* list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
data_ds = ex_data.test_ds, data_ds = ex_data.test_ds,
...@@ -1092,8 +1113,9 @@ if __name__ == "__main__": ...@@ -1092,8 +1113,9 @@ if __name__ == "__main__":
high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test) high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test)) print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test))
fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD)) fpath = test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS) ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS, num_scenes = num_test_scenes)
plt.show() plt.show()
"""
scene = os.path.basename(test_corr)[:17] scene = os.path.basename(test_corr)[:17]
scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr))) scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
fname =scene+'-'+scene_version fname =scene+'-'+scene_version
...@@ -1102,6 +1124,6 @@ if __name__ == "__main__": ...@@ -1102,6 +1124,6 @@ if __name__ == "__main__":
writeTFRewcordsImageTiles(test_corr, img_filenameTFR) writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
print_time("Done") print_time("Done")
pass pass
"""
pass pass
nn_ds_neibs9.py
View file @ 94636179
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment