Next versions

explore_data.py

@@ -625,7 +625,16 @@ class ExploreData:
             
         if set_ds is None:
             set_ds = self.train_ds
-            
+        try:
+            os.makedirs(os.path.dirname(tfr_filename))
+            print("Created directory "+os.path.dirname(tfr_filename))
+        except:
+            print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
+            pass
+        #skip writing if file exists - it will be possible to continue or run several instances
+        if os.path.exists(tfr_filename):
+            print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
+            return     
         writer = tf.python_io.TFRecordWriter(tfr_filename)
 #$        files_list = [self.files_train, self.files_test][test_set]
         seed_list = np.arange(len(files_list))
@@ -666,7 +675,7 @@ class ExploreData:
                 example = tf.train.Example(features=tf.train.Features(feature=d_feature))
                 writer.write(example.SerializeToString())
             if (self.debug_level > 0):
-                print("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        
+                print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        
         writer.close()
         sys.stdout.flush()        
 
@@ -742,38 +751,30 @@ if __name__ == "__main__":
   try:
       topdir_train = sys.argv[1]
   except IndexError:
-      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
+#      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
+      topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
   try:
       topdir_test = sys.argv[2]
   except IndexError:
-      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
+#      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
+      topdir_test =  "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
       
   try:
       pathTFR =     sys.argv[3]
   except IndexError:
-      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
 
   try:
       ml_subdir =   sys.argv[4]
   except IndexError:
-      ml_subdir =   "ml"
-      
+#      ml_subdir =   "ml"
+      ml_subdir =   "mlr32_18a"
       
-      
-#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
   test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
-  scene = os.path.basename(test_corr)[:17]
-  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
-  fname =scene+'-'+scene_version 
-  img_filenameTFR = os.path.join(pathTFR,'img',fname)        
-  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
-  
-  pass
-  
-  exit(0)
       
   #Parameters to generate neighbors data. Set radius to 0 to generate single-tile     
-  RADIUS = 1
+  RADIUS = 2 # 5x5
   MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
   VARIANCE_THRESHOLD = 1.5
   NUM_TRAIN_SETS = 8
@@ -944,6 +945,13 @@ if __name__ == "__main__":
       fpath =  test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
       ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
   plt.show()
-  
+#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#  test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
+  scene = os.path.basename(test_corr)[:17]
+  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
+  fname =scene+'-'+scene_version 
+  img_filenameTFR = os.path.join(pathTFR,'img',fname)        
+  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
   pass
+  exit(0)
     

nn_ds_neibs2.py

@@ -31,7 +31,7 @@ FILES_PER_SCENE =    5 # number of random offset files for the scene to select f
 #MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
 #MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
 #MAX_EPOCH =        500
-LR =               1e-4 # learning rate
+LR =               1e-3 # learning rate
 LR100 =            1e-4
 USE_CONFIDENCE =     False
 ABSOLUTE_DISPARITY = False # True # False # True # False
@@ -47,8 +47,8 @@ RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batche
 #BATCH_SIZE =       1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
 BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
 SHUFFLE_EPOCH =    True
-NET_ARCH1 =          0 # 6 #0 # 4 # 3 # overwrite with argv?
-NET_ARCH2 =          0 # 6 # 0 # 3 # overwrite with argv?
+NET_ARCH1 =          0 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =          3 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
 ONLY_TILE =          None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
 ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
 
@@ -331,7 +331,7 @@ except IndexError:
 train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
 """
 files_img =        ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords']
-result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R.npy'
+result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-3.npy'
 
 files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords",
                     "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords",
@@ -364,11 +364,11 @@ import tensorflow as tf
 import tensorflow.contrib.slim as slim
 
 
-#try:
-eval_results(result_file, ABSOLUTE_DISPARITY)
-exit(0)
-#except:
-#    pass
+try:
+    eval_results(result_file, ABSOLUTE_DISPARITY)
+#exit(0)
+except:
+    pass
 
 datasets_img = []
 for fpath in files_img:
@@ -503,7 +503,7 @@ dataset_img_size //= BATCH_SIZE
 
 #print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
 dataset_tt = dataset_tt.batch(BATCH_SIZE)
-#dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
 iterator_tt = dataset_tt.make_initializable_iterator()
 next_element_tt = iterator_tt.get_next()
 #print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))