Commit 464cacc9 authored by Andrey Filippov's avatar Andrey Filippov
Browse files

Next versions

parent bb3cc9de

explore_data.py

+29 −21
Original line number Diff line number Diff line
@@ -625,7 +625,16 @@ class ExploreData: 
            

        if set_ds is None:

            set_ds = self.train_ds

            

        try:

            os.makedirs(os.path.dirname(tfr_filename))

            print("Created directory "+os.path.dirname(tfr_filename))

        except:

            print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")

            pass

        #skip writing if file exists - it will be possible to continue or run several instances

        if os.path.exists(tfr_filename):

            print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")

            return     

        writer = tf.python_io.TFRecordWriter(tfr_filename)

#$        files_list = [self.files_train, self.files_test][test_set]

        seed_list = np.arange(len(files_list))
@@ -666,7 +675,7 @@ class ExploreData: 
                example = tf.train.Example(features=tf.train.Features(feature=d_feature))

                writer.write(example.SerializeToString())

            if (self.debug_level > 0):

                print("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        

                print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        

        writer.close()

        sys.stdout.flush()
@@ -742,38 +751,30 @@ if __name__ == "__main__": 
  try:

      topdir_train = sys.argv[1]

  except IndexError:

      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"

#      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"

      topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"

  try:

      topdir_test = sys.argv[2]

  except IndexError:

      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"

#      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"

      topdir_test =  "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"

      

  try:

      pathTFR =     sys.argv[3]

  except IndexError:

      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"

#      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"

      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"



  try:

      ml_subdir =   sys.argv[4]

  except IndexError:

      ml_subdir =   "ml"

      

#      ml_subdir =   "ml"

      ml_subdir =   "mlr32_18a"

      

      

#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"

  test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'

  scene = os.path.basename(test_corr)[:17]

  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))

  fname =scene+'-'+scene_version 

  img_filenameTFR = os.path.join(pathTFR,'img',fname)        

  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)

  

  pass

  

  exit(0)

      

  #Parameters to generate neighbors data. Set radius to 0 to generate single-tile     

  RADIUS = 1

  RADIUS = 2 # 5x5

  MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9

  VARIANCE_THRESHOLD = 1.5

  NUM_TRAIN_SETS = 8
@@ -944,6 +945,13 @@ if __name__ == "__main__": 
      fpath =  test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))

      ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)

  plt.show()

  

#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"

#  test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'

  scene = os.path.basename(test_corr)[:17]

  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))

  fname =scene+'-'+scene_version 

  img_filenameTFR = os.path.join(pathTFR,'img',fname)        

  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)

  pass

  exit(0)

explore_data1.py

0 → 100644
+1067 −0

File added.

Preview size limit exceeded, changes collapsed.

explore_data2.py

0 → 100644
+1107 −0

File added.

Preview size limit exceeded, changes collapsed.

nn_ds_neibs2.py

+10 −10
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ FILES_PER_SCENE = 5 # number of random offset files for the scene to select f 
#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 

#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch

#MAX_EPOCH =        500

LR =               1e-4 # learning rate

LR =               1e-3 # learning rate

LR100 =            1e-4

USE_CONFIDENCE =     False

ABSOLUTE_DISPARITY = False # True # False # True # False
@@ -47,8 +47,8 @@ RUN_TOT_AVG = 100 # last batches to average. Epoch is 307 training batche 
#BATCH_SIZE =       1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches

BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches

SHUFFLE_EPOCH =    True

NET_ARCH1 =          0 # 6 #0 # 4 # 3 # overwrite with argv?

NET_ARCH2 =          0 # 6 # 0 # 3 # overwrite with argv?

NET_ARCH1 =          0 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?

NET_ARCH2 =          3 # 2 #0 # 6 # 0 # 3 # overwrite with argv?

ONLY_TILE =          None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)

ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements
@@ -331,7 +331,7 @@ except IndexError: 
train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"

"""

files_img =        ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords']

result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R.npy'

result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-3.npy'



files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords",

                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords",
@@ -364,11 +364,11 @@ import tensorflow as tf 
import tensorflow.contrib.slim as slim





#try:

try:

    eval_results(result_file, ABSOLUTE_DISPARITY)

exit(0)

#except:

#    pass

#exit(0)

except:

    pass



datasets_img = []

for fpath in files_img:
@@ -503,7 +503,7 @@ dataset_img_size //= BATCH_SIZE 


#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))

dataset_tt = dataset_tt.batch(BATCH_SIZE)

#dataset_tt = dataset_tt.prefetch(BATCH_SIZE)

dataset_tt = dataset_tt.prefetch(BATCH_SIZE)

iterator_tt = dataset_tt.make_initializable_iterator()

next_element_tt = iterator_tt.get_next()

#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))

nn_ds_neibs3.py

0 → 100644
+1043 −0

File added.

Preview size limit exceeded, changes collapsed.