Merge branch 'master' of git.elphel.com:Elphel/python3-imagej-tiff

explore_data.py

@@ -625,7 +625,16 @@ class ExploreData:
             
         if set_ds is None:
             set_ds = self.train_ds
-            
+        try:
+            os.makedirs(os.path.dirname(tfr_filename))
+            print("Created directory "+os.path.dirname(tfr_filename))
+        except:
+            print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
+            pass
+        #skip writing if file exists - it will be possible to continue or run several instances
+        if os.path.exists(tfr_filename):
+            print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
+            return     
         writer = tf.python_io.TFRecordWriter(tfr_filename)
 #$        files_list = [self.files_train, self.files_test][test_set]
         seed_list = np.arange(len(files_list))
@@ -666,7 +675,7 @@ class ExploreData:
                 example = tf.train.Example(features=tf.train.Features(feature=d_feature))
                 writer.write(example.SerializeToString())
             if (self.debug_level > 0):
-                print("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        
+                print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        
         writer.close()
         sys.stdout.flush()        
 
@@ -742,38 +751,30 @@ if __name__ == "__main__":
   try:
       topdir_train = sys.argv[1]
   except IndexError:
-      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
+#      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
+      topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
   try:
       topdir_test = sys.argv[2]
   except IndexError:
-      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
+#      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
+      topdir_test =  "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
       
   try:
       pathTFR =     sys.argv[3]
   except IndexError:
-      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
 
   try:
       ml_subdir =   sys.argv[4]
   except IndexError:
-      ml_subdir =   "ml"
-      
+#      ml_subdir =   "ml"
+      ml_subdir =   "mlr32_18a"
       
-      
-#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
   test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
-  scene = os.path.basename(test_corr)[:17]
-  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
-  fname =scene+'-'+scene_version 
-  img_filenameTFR = os.path.join(pathTFR,'img',fname)        
-  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
-  
-  pass
-  
-  exit(0)
-
+      
   #Parameters to generate neighbors data. Set radius to 0 to generate single-tile     
-  RADIUS = 1
+  RADIUS = 2 # 5x5
   MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
   VARIANCE_THRESHOLD = 1.5
   NUM_TRAIN_SETS = 8
@@ -944,6 +945,13 @@ if __name__ == "__main__":
       fpath =  test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
       ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
   plt.show()
-  
+#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#  test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
+  scene = os.path.basename(test_corr)[:17]
+  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
+  fname =scene+'-'+scene_version 
+  img_filenameTFR = os.path.join(pathTFR,'img',fname)        
+  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
   pass
+  exit(0)
     

explore_data1.py

+#!/usr/bin/env python3
+from numpy import float64
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+import os
+import sys
+import glob
+import imagej_tiff as ijt
+import numpy as np
+import resource
+import timeit
+import matplotlib.pyplot as plt
+from scipy.ndimage.filters import gaussian_filter
+import time
+import tensorflow as tf
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+    
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end)
+    TIME_LAST = t
+
+def _dtype_feature(ndarray):
+    """match appropriate tf.train.Feature class with dtype of ndarray. """
+    assert isinstance(ndarray, np.ndarray)
+    dtype_ = ndarray.dtype
+    if dtype_ == np.float64 or dtype_ == np.float32:
+        return lambda array: tf.train.Feature(float_list=tf.train.FloatList(value=array))
+    elif dtype_ == np.int64:
+        return lambda array: tf.train.Feature(int64_list=tf.train.Int64List(value=array))
+    else:  
+        raise ValueError("The input should be numpy ndarray. \
+                           Instead got {}".format(ndarray.dtype))
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecordDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+    corr2d_list=[]
+    target_disparity_list=[]
+    gt_ds_list = []
+    for string_record in record_iterator:
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        corr2d_list.append(np.array(example.features.feature['corr2d'] .float_list .value))
+        target_disparity_list.append(np.array(example.features.feature['target_disparity'] .float_list .value[0]))
+        gt_ds_list.append(np.array(example.features.feature['gt_ds'] .float_list .value))
+    corr2d=            np.array(corr2d_list)
+    target_disparity = np.array(target_disparity_list)
+    gt_ds =            np.array(gt_ds_list)
+    return corr2d, target_disparity, gt_ds   
+
+def writeTFRewcordsImageTiles(img_path, tfr_filename): # test_set=False):
+#        train_filename = 'train.tfrecords'  # address to save the TFRecords file
+      # open the TFRecords file
+      num_tiles = 242*324 # fixme
+      all_image_tiles = np.array(range(num_tiles))
+      corr_layers =  ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
+      img =          ijt.imagej_tiff(test_corr, corr_layers, all_image_tiles)
+
+      corr2d =           img.corr2d.reshape((num_tiles,-1))
+      target_disparity = img.target_disparity.reshape((num_tiles,-1))
+      gt_ds =            img.gt_ds.reshape((num_tiles,-1))
+
+      if not  '.tfrecords' in tfr_filename:
+          tfr_filename += '.tfrecords'
+
+      tfr_filename=tfr_filename.replace(' ','_')
+      try:
+          os.makedirs(os.path.dirname(tfr_filename))
+      except:
+          pass     
+          
+      writer = tf.python_io.TFRecordWriter(tfr_filename)
+      dtype_feature_corr2d =   _dtype_feature(corr2d)
+      dtype_target_disparity = _dtype_feature(target_disparity)
+      dtype_feature_gt_ds =    _dtype_feature(gt_ds)
+      for i in range(num_tiles):
+          x = corr2d[i].astype(np.float32)
+          y = target_disparity[i].astype(np.float32)
+          z = gt_ds[i].astype(np.float32)
+          d_feature = {'corr2d':          dtype_feature_corr2d(x),
+                       'target_disparity':dtype_target_disparity(y),
+                       'gt_ds':           dtype_feature_gt_ds(z)}
+          example = tf.train.Example(features=tf.train.Features(feature=d_feature))
+          writer.write(example.SerializeToString())
+          pass
+      writer.close()
+      sys.stdout.flush()        
+
+
+
+class ExploreData:
+    PATTERN = "*-DSI_COMBO.tiff"
+#    ML_DIR = "ml"
+    ML_PATTERN = "*-ML_DATA*OFFS*.tiff"
+    """
+1527182801_296892-ML_DATARND-32B-O-FZ0.05-OFFS-0.20000_0.20000.tiff
+    """
+    def getComboList(self, top_dir):
+#        patt = "*-DSI_COMBO.tiff"
+        tlist = []
+        for i in range(5):
+            pp = top_dir#) ,'**', patt) # works
+            for j in range (i):
+                pp = os.path.join(pp,'*')
+            pp = os.path.join(pp, ExploreData.PATTERN)
+            tlist += glob.glob(pp)
+            if (self.debug_level > 0):    
+                print (pp+" "+str(len(tlist)))
+        if (self.debug_level > 0):    
+            print("Found "+str(len(tlist))+" combo DSI files in "+top_dir+" :")
+            if (self.debug_level > 1):    
+                print("\n".join(tlist))
+        return tlist
+    
+    def loadComboFiles(self, tlist):
+        indx = 0
+        images = []
+        if (self.debug_level>2):
+            print(str(resource.getrusage(resource.RUSAGE_SELF)))
+        for combo_file in tlist:
+            tiff = ijt.imagej_tiff(combo_file,['disparity_rig','strength_rig'])
+            if not indx:
+                images = np.empty((len(tlist), tiff.image.shape[0],tiff.image.shape[1],tiff.image.shape[2]), tiff.image.dtype)
+            images[indx] = tiff.image
+            if (self.debug_level>2):
+                print(str(indx)+": "+str(resource.getrusage(resource.RUSAGE_SELF)))
+            indx += 1
+        return images
+    
+    def getHistogramDSI(
+            self, 
+            list_rds,
+            disparity_bins =    1000,
+            strength_bins =      100,
+            disparity_min_drop =  -0.1,
+            disparity_min_clip =  -0.1,
+            disparity_max_drop = 100.0,
+            disparity_max_clip = 100.0,
+            strength_min_drop =    0.1,
+            strength_min_clip =    0.1,
+            strength_max_drop =    1.0,
+            strength_max_clip =    0.9,
+            normalize =           True,
+            no_histogram =        False            
+            ):
+        good_tiles_list=[]
+        for combo_rds in list_rds:
+            good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
+            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+                ds = combo_rds[ids]
+                disparity = ds[...,0]
+                strength =  ds[...,1]
+                good_tiles[ids] =  disparity >= disparity_min_drop
+                good_tiles[ids] &= disparity <= disparity_max_drop
+                good_tiles[ids] &= strength >=  strength_min_drop
+                good_tiles[ids] &= strength <=  strength_max_drop
+                
+                disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
+                strength =  np.nan_to_num(strength, copy = False)  # likely should never happen
+                np.clip(disparity, disparity_min_clip, disparity_max_clip, out = disparity)
+                np.clip(strength, strength_min_clip, strength_max_clip, out = strength)
+            good_tiles_list.append(good_tiles)
+        combo_rds = np.concatenate(list_rds)
+        hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
+            x =      combo_rds[...,1].flatten(),
+            y =      combo_rds[...,0].flatten(),
+            bins=    (strength_bins, disparity_bins),
+            range=   ((strength_min_clip,strength_max_clip),(disparity_min_clip,disparity_max_clip)),
+            normed=  normalize,
+            weights= np.concatenate(good_tiles_list).flatten())
+        for i, combo_rds in enumerate(list_rds):
+            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+                combo_rds[ids][...,1]*= good_tiles_list[i][ids]
+        return hist, xedges, yedges
+    
+    
+    def __init__(self,
+               topdir_train,
+               topdir_test,
+               ml_subdir,  
+               debug_level =          0,
+               disparity_bins =    1000,
+               strength_bins =      100,
+               disparity_min_drop =  -0.1,
+               disparity_min_clip =  -0.1,
+               disparity_max_drop = 100.0,
+               disparity_max_clip = 100.0,
+               strength_min_drop =    0.1,
+               strength_min_clip =    0.1,
+               strength_max_drop =    1.0,
+               strength_max_clip =    0.9,
+               hist_sigma =           2.0,  # Blur log histogram
+               hist_cutoff=           0.001 #  of maximal  
+               ):
+    # file name
+        self.debug_level = debug_level
+        #self.testImageTiles()    
+        self.disparity_bins =     disparity_bins
+        self.strength_bins =      strength_bins
+        self.disparity_min_drop = disparity_min_drop
+        self.disparity_min_clip = disparity_min_clip
+        self.disparity_max_drop = disparity_max_drop
+        self.disparity_max_clip = disparity_max_clip
+        self.strength_min_drop =  strength_min_drop
+        self.strength_min_clip =  strength_min_clip
+        self.strength_max_drop =  strength_max_drop
+        self.strength_max_clip =  strength_max_clip
+        self.hist_sigma =         hist_sigma # Blur log histogram
+        self.hist_cutoff=         hist_cutoff #  of maximal  
+        self.pre_log_offs =       0.001 # of histogram maximum
+        self.good_tiles =         None
+        self.files_train =        self.getComboList(topdir_train)
+        self.files_test =         self.getComboList(topdir_test)
+        
+        self.train_ds =           self.loadComboFiles(self.files_train)
+        self.test_ds =            self.loadComboFiles(self.files_test)
+        
+        self.num_tiles = self.train_ds.shape[1]*self.train_ds.shape[2] 
+        self.hist, xedges, yedges = self.getHistogramDSI(
+                list_rds =           [self.train_ds,self.test_ds], # combo_rds,
+                disparity_bins =     self.disparity_bins,
+                strength_bins =      self.strength_bins,
+                disparity_min_drop = self.disparity_min_drop,
+                disparity_min_clip = self.disparity_min_clip,
+                disparity_max_drop = self.disparity_max_drop,
+                disparity_max_clip = self.disparity_max_clip,
+                strength_min_drop =  self.strength_min_drop,
+                strength_min_clip =  self.strength_min_clip,
+                strength_max_drop =  self.strength_max_drop,
+                strength_max_clip =  self.strength_max_clip,
+                normalize =          True,
+                no_histogram =       False
+           )
+        log_offset = self.pre_log_offs * self.hist.max()
+        h_cutoff =   hist_cutoff * self.hist.max()
+        lhist =         np.log(self.hist + log_offset)
+        blurred_lhist = gaussian_filter(lhist, sigma = self.hist_sigma)
+        self.blurred_hist  = np.exp(blurred_lhist) - log_offset
+        self.good_tiles =  self.blurred_hist >= h_cutoff
+        self.blurred_hist *= self.good_tiles # set bad ones to zero 
+
+    def exploreNeibs(self,
+                     data_ds, # disparity/strength data for all files (train or test)
+                     radius,  # how far to look from center each side ( 1- 3x3, 2 - 5x5)
+                     disp_thesh = 5.0): # reduce effective variance for higher disparities
+        """
+        For each tile calculate difference between max and min among neighbors and number of qualifying neighbors (bad cewnter is not removed)
+        """
+        disp_min =   np.empty_like(data_ds[...,0], dtype = np.float)
+        disp_max =   np.empty_like(disp_min, dtype = np.float)
+        tile_neibs = np.zeros_like(disp_min, dtype = np.int)
+        dmin = data_ds[...,0].min()
+        dmax = data_ds[...,0].max()
+        good_tiles = self.getBB(data_ds) >= 0
+        side = 2 * radius + 1
+        for nf, ds in enumerate(data_ds):
+            disp = ds[...,0] 
+            height = disp.shape[0]
+            width = disp.shape[1]
+            bad_max = np.ones((height+side, width+side),  dtype=float) * dmax
+            bad_min = np.ones((height+side, width+side),  dtype=float) * dmin
+            good =    np.zeros((height+side, width+side), dtype=int)
+            #Assign centers of the array, replace bad tiles with max/min (so they will not change min/max) 
+            bad_max[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmax)
+            bad_min[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmin)
+            good   [radius:height+radius,radius:width+radius] = good_tiles[nf]
+            disp_min  [nf,...] = disp 
+            disp_max  [nf,...] = disp
+            tile_neibs[nf,...] = good_tiles[nf]
+            for offset_y in range(-radius, radius+1):
+                oy = offset_y+radius
+                for offset_x in range(-radius, radius+1):
+                    ox = offset_x+radius
+                    if offset_y or offset_x: # Skip center - already copied
+                        np.minimum(disp_min[nf], bad_max[oy:oy+height, ox:ox+width], out=disp_min[nf])
+                        np.maximum(disp_max[nf], bad_min[oy:oy+height, ox:ox+width], out=disp_max[nf])
+                        tile_neibs[nf] +=  good[oy:oy+height, ox:ox+width]
+                        pass
+                    pass
+                pass
+            pass
+        
+        #disp_thesh
+        disp_avar = disp_max - disp_min
+        disp_rvar = disp_avar * disp_thesh / np.maximum(disp_max, 0.001) # removing division by 0 error - those tiles will be anyway discarded 
+        disp_var = np.select([disp_max >= disp_thesh, disp_max < disp_thesh],[disp_rvar,disp_avar])
+        return disp_var, tile_neibs
+
+    def assignBatchBins(self,
+                        disp_bins,
+                        str_bins,
+                        files_per_scene = 5,   # not used here, will be used when generating batches
+                        min_batch_choices=10,  # not used here, will be used when generating batches
+                        max_batch_files = 10): # not used here, will be used when generating batches
+        """
+        for each disparity/strength combination (self.disparity_bins * self.strength_bins = 1000*100) provide number of "large"
+        variable-size disparity/strength bin, or -1 if this disparity/strength combination does not seem right
+        """
+        self.files_per_scene = files_per_scene
+        self.min_batch_choices=min_batch_choices
+        self.max_batch_files = max_batch_files
+        
+        hist_to_batch =       np.zeros((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
+        hist_to_batch_multi = np.ones((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
+        scale_hist= (disp_bins * str_bins)/self.blurred_hist.sum()
+        norm_b_hist =     self.blurred_hist * scale_hist
+        disp_list = [] # last disparity hist 
+#        disp_multi = [] # number of disp rows to fit
+        disp_run_tot = 0.0
+        disp_batch = 0
+        disp=0
+        num_batch_bins = disp_bins * str_bins
+        disp_hist = np.linspace(0, num_batch_bins, disp_bins+1)
+        batch_index = 0
+        num_members = np.zeros((num_batch_bins,),int)
+        while disp_batch < disp_bins:
+            #disp_multi.append(1)
+#        while (disp < self.disparity_bins):
+#            disp_target_tot =disp_hist[disp_batch+1]
+            disp_run_tot_new = disp_run_tot
+            disp0 = disp # start disaprity matching disp_run_tot 
+            while (disp_run_tot_new < disp_hist[disp_batch+1]) and (disp < self.disparity_bins):
+                disp_run_tot_new += norm_b_hist[:,disp].sum()
+                disp+=1;
+                disp_multi = 1
+                while   (disp_batch < (disp_bins - 1)) and (disp_run_tot_new >= disp_hist[disp_batch+2]):
+                    disp_batch += 1 # only if large disp_bins and very high hist value
+                    disp_multi += 1
+            # now  disp_run_tot - before this batch disparity col
+            str_bins_corr = str_bins * disp_multi # if too narrow disparity column - multiply number of strength columns
+            str_bins_corr_last = str_bins_corr -1
+            str_hist = np.linspace(disp_run_tot, disp_run_tot_new, str_bins_corr + 1)
+            str_run_tot_new = disp_run_tot
+#            str_batch = 0
+            str_index=0
+#            wide_col = norm_b_hist[:,disp0:disp] #disp0 - first column, disp - last+ 1
+            #iterate in linescan along the column
+            for si in range(self.strength_bins):
+                for di in range(disp0, disp,1):
+                    if norm_b_hist[si,di] > 0.0 :
+                        str_run_tot_new += norm_b_hist[si,di]
+                        # do not increment after last to avoid precision issues 
+                        if (batch_index < num_batch_bins) and (num_members[batch_index] > 0) and (str_index < str_bins_corr_last) and (str_run_tot_new > str_hist[str_index+1]):
+                            batch_index += 1
+                            str_index +=   1
+                        if batch_index < num_batch_bins :     
+                            hist_to_batch[si,di] = batch_index
+                            num_members[batch_index] += 1
+                        else:
+                            pass
+                    else:
+                        hist_to_batch[si,di] = -1
+                        
+            batch_index += 1 # it was not incremented afterthe last in the column to avoid rounding error 
+            disp_batch += 1
+            disp_run_tot = disp_run_tot_new
+            pass
+        self.hist_to_batch = hist_to_batch
+        return hist_to_batch        
+
+    def getBB(self, data_ds):
+        """
+        for each file, each tile get histogram index (or -1 for bad tiles)
+        """
+        hist_to_batch = self.hist_to_batch
+        files_batch_list = []
+        disp_step = ( self.disparity_max_clip - self.disparity_min_clip )/ self.disparity_bins 
+        str_step =  ( self.strength_max_clip -  self.strength_min_clip )/ self.strength_bins
+        bb = np.empty_like(data_ds[...,0],dtype=int)
+        for findx in range(data_ds.shape[0]):
+            ds = data_ds[findx]
+            gt = ds[...,1] > 0.0 # OK
+            db = (((ds[...,0] - self.disparity_min_clip)/disp_step).astype(int))*gt
+            sb = (((ds[...,1] - self.strength_min_clip)/ str_step).astype(int))*gt
+            np.clip(db, 0, self.disparity_bins-1, out = db)
+            np.clip(sb, 0, self.strength_bins-1, out = sb)
+            bb[findx] = (self.hist_to_batch[sb.reshape(self.num_tiles),db.reshape(self.num_tiles)])   .reshape(db.shape[0],db.shape[1]) + (gt -1)
+        return bb
+
+    def makeBatchLists(self,
+            data_ds =      None, # (disparity,strength) per scene, per tile
+            disp_var =     None, # difference between maximal and minimal disparity for each scene, each tile
+            disp_neibs =   None, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+            min_var =      None, # Minimal tile variance to include
+            max_var =      None, # Maximal tile variance to include
+            min_neibs =    None):# Minimal number of valid tiles to include
+        if data_ds is None:
+             data_ds =      self.train_ds
+        hist_to_batch = self.hist_to_batch
+        num_batch_tiles = np.empty((data_ds.shape[0],self.hist_to_batch.max()+1),dtype = int) 
+        bb = self.getBB(data_ds)
+        use_neibs = not ((disp_var is None) or (disp_neibs is None) or (min_var is None) or (max_var is None) or (min_neibs is None))
+        list_of_file_lists=[]
+        for findx in range(data_ds.shape[0]):
+            foffs = findx * self.num_tiles 
+            lst = []
+            for i in range (self.hist_to_batch.max()+1):
+                lst.append([])
+#            bb1d = bb[findx].reshape(self.num_tiles)
+            if use_neibs:    
+                disp_var_tiles =   disp_var[findx].reshape(self.num_tiles)
+                disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
+            for n, indx in enumerate(bb[findx].reshape(self.num_tiles)):
+                if indx >= 0:
+                    if use_neibs:
+#                        disp_var_tiles =   disp_var[findx].reshape(self.num_tiles)
+#                        disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
+                        if disp_neibs_tiles[n] < min_neibs:
+                            continue # too few neighbors
+                        if not disp_var_tiles[n] >= min_var:
+                            continue #too small variance 
+                        if not disp_var_tiles[n] <  max_var:
+                            continue #too large variance 
+                    lst[indx].append(foffs + n)
+            lst_arr=[]
+            for i,l in enumerate(lst):
+#                lst_arr.append(np.array(l,dtype = int))
+                lst_arr.append(l)
+                num_batch_tiles[findx,i] = len(l)
+            list_of_file_lists.append(lst_arr)
+        self.list_of_file_lists= list_of_file_lists
+        self.num_batch_tiles =   num_batch_tiles
+        return list_of_file_lists, num_batch_tiles
+    #todo: only use other files if there are no enough choices in the main file!
+    
+    
+    def augmentBatchFileIndices(self,
+                                 seed_index,
+                                 min_choices=None,
+                                 max_files = None,
+                                 set_ds = None
+                                 ):
+        if min_choices is None:
+            min_choices = self.min_batch_choices 
+        if max_files is None:
+            max_files =  self.max_batch_files
+        if set_ds is None:
+            set_ds = self.train_ds
+        full_num_choices = self.num_batch_tiles[seed_index].copy()
+        flist = [seed_index]
+        all_choices = list(range(self.num_batch_tiles.shape[0]))
+        all_choices.remove(seed_index)
+        for _ in range (max_files-1):
+            if full_num_choices.min() >= min_choices:
+                break
+            findx = np.random.choice(all_choices)
+            flist.append(findx)
+            all_choices.remove(findx)
+            full_num_choices += self.num_batch_tiles[findx]
+
+        file_tiles_sparse = [[] for _ in set_ds] #list of empty lists for each train scene (will be sparse) 
+        for nt in range(self.num_batch_tiles.shape[1]): #number of tiles per batch (not counting ml file variant)
+            tl = []
+            nchoices = 0
+            for findx in flist:
+                if (len(self.list_of_file_lists[findx][nt])):
+                    tl.append(self.list_of_file_lists[findx][nt])
+                nchoices+= self.num_batch_tiles[findx][nt]
+                if nchoices >= min_choices: # use minimum of extra files
+                    break;
+            tile = np.random.choice(np.concatenate(tl))
+#            print (nt, tile, tile//self.num_tiles, tile % self.num_tiles)
+            if not type (tile) is np.int64:
+                print("tile=",tile)
+            file_tiles_sparse[tile//self.num_tiles].append(tile % self.num_tiles)
+        file_tiles = []
+        for findx in flist:
+            file_tiles.append(np.sort(np.array(file_tiles_sparse[findx],dtype=int))) 
+        return flist, file_tiles # file indices, list if tile indices for each file   
+            
+            
+               
+                
+    def getMLList(self, ml_subdir, flist):
+        ml_list = []
+        for fn in flist:
+            ml_patt = os.path.join(os.path.dirname(fn), ml_subdir, ExploreData.ML_PATTERN)
+            ml_list.append(glob.glob(ml_patt))
+##        self.ml_list = ml_list
+        return ml_list
+            
+    def getBatchData(
+            self,
+            flist,
+            tiles,
+            ml_list,
+            ml_num = None ): # 0 - use all ml files for the scene, >0 select random number
+        if ml_num is None:
+            ml_num = self.files_per_scene
+        ml_all_files = []
+        for findx in flist:
+            mli =  list(range(len(ml_list[findx])))
+            if (ml_num > 0) and (ml_num < len(mli)):
+                mli_left = mli
+                mli = []
+                for _ in range(ml_num):
+                    ml = np.random.choice(mli_left)
+                    mli.append(ml)
+                    mli_left.remove(ml)
+            ml_files = []
+            for ml_index in mli:
+                ml_files.append(ml_list[findx][ml_index])
+            ml_all_files.append(ml_files)        
+                    
+        return ml_all_files
+    
+    def prepareBatchData(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
+        if min_choices is None:
+            min_choices = self.min_batch_choices
+        if max_files is None:
+            max_files = self.max_batch_files
+        if ml_num is None:
+            ml_num = self.files_per_scene
+        if set_ds is None:
+            set_ds = self.train_ds
+        tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
+        height = set_ds.shape[1]
+        width =  set_ds.shape[2]
+        width_m1 = width-1
+        height_m1 = height-1
+#        set_ds = [self.train_ds, self.test_ds][test_set]            
+        corr_layers =  ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
+        flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
+        
+#        ml_all_files = self.getBatchData(flist, tiles, ml_list,  ml_num) # 0 - use all ml files for the scene, >0 select random number
+        ml_all_files = self.getBatchData(flist, tiles, ml_list,  0) # ml_num) # 0 - use all ml files for the scene, >0 select random number
+        if self.debug_level > 1:
+            print ("==============",seed_index, flist)
+            for i, findx in enumerate(flist):
+                print(i,"\n".join(ml_all_files[i])) 
+                print(tiles[i]) 
+        total_tiles = 0
+        for i, t in enumerate(tiles):
+##          total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
+            total_tiles += len(t) # tiles per scene * offset files per scene
+        if self.debug_level > 1:
+            print("Tiles in the batch=",total_tiles)
+        corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
+        gt_ds_batch =            np.empty((total_tiles * tiles_in_sample, 2), dtype=float) 
+        target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ),  dtype=float) 
+        start_tile = 0
+        for nscene, scene_files in enumerate(ml_all_files):
+            '''
+            Create tiles list including neighbors
+            '''
+            full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
+            indx = 0;
+            for i, nt in enumerate(tiles[nscene]):
+                ty = nt // width
+                tx = nt % width
+                for dy in range (-radius, radius+1):
+                    y = np.clip(ty+dy,0,height_m1)
+                    for dx in range (-radius, radius+1):
+                        x = np.clip(tx+dx,0,width_m1)
+                        full_tiles[indx] = y * width + x
+                        indx += 1
+            """
+            Assign tiles to several correlation files
+            """
+            file_tiles = []
+            file_indices = []
+            for f in scene_files:
+                file_tiles.append([])
+            num_scene_files = len(scene_files)    
+            for t in full_tiles:
+                fi = np.random.randint(0, num_scene_files)
+                file_tiles[fi].append(t)
+                file_indices.append(fi)
+            corr2d_list =           []
+            target_disparity_list = []
+            gt_ds_list =            []
+            for fi, path in enumerate (scene_files):
+                img = ijt.imagej_tiff(path, corr_layers, tile_list=file_tiles[fi])
+                corr2d_list.append          (img.corr2d)
+                target_disparity_list.append(img.target_disparity)
+                gt_ds_list.append           (img.gt_ds)
+            img_indices = [0] * len(scene_files)
+            for i, fi in enumerate(file_indices):
+                ti = img_indices[fi]
+                img_indices[fi] += 1
+                if corr2d_batch is None:
+                    corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d_list[fi].shape[-1]))
+                gt_ds_batch            [start_tile] = gt_ds_list[fi][ti]
+                target_disparity_batch [start_tile] = target_disparity_list[fi][ti]
+                corr2d_batch           [start_tile] = corr2d_list[fi][ti]
+                start_tile +=  1
+            """
+             Sometimes get bad tile in ML file that was not bad in COMBO-DSI
+             Need to recover
+             np.argwhere(np.isnan(target_disparity_batch))                 
+            """
+        bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
+        if (len(bad_tiles)>0):
+            print ("*** Got %d bad tiles in a batch, no code to replace :-("%(len(bad_tiles)))
+            # for now - just repeat some good tile
+            """
+            for ibt in bad_tiles:
+                while np.isnan(target_disparity_batch[ibt]):
+                    irt = np.random.randint(0,total_tiles)
+                    if not np.isnan(target_disparity_batch[irt]):
+                        target_disparity_batch[ibt] = target_disparity_batch[irt]
+                        corr2d_batch[ibt] = corr2d_batch[irt]
+                        gt_ds_batch[ibt] = gt_ds_batch[irt]
+                        break
+            print (" done replacing")
+            """
+        self.corr2d_batch =           corr2d_batch
+        self.target_disparity_batch = target_disparity_batch
+        self.gt_ds_batch =            gt_ds_batch
+        return corr2d_batch, target_disparity_batch, gt_ds_batch
+
+    def prepareBatchDataOld(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
+        if min_choices is None:
+            min_choices = self.min_batch_choices
+        if max_files is None:
+            max_files = self.max_batch_files
+        if ml_num is None:
+            ml_num = self.files_per_scene
+        if set_ds is None:
+            set_ds = self.train_ds
+        tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
+        height = set_ds.shape[1]
+        width =  set_ds.shape[2]
+        width_m1 = width-1
+        height_m1 = height-1
+#        set_ds = [self.train_ds, self.test_ds][test_set]            
+        corr_layers =  ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
+        flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
+        
+        ml_all_files = self.getBatchData(flist, tiles, ml_list,  ml_num) # 0 - use all ml files for the scene, >0 select random number
+        if self.debug_level > 1:
+            print ("==============",seed_index, flist)
+            for i, findx in enumerate(flist):
+                print(i,"\n".join(ml_all_files[i])) 
+                print(tiles[i]) 
+        total_tiles = 0
+        for i, t in enumerate(tiles):
+            total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
+        if self.debug_level > 1:
+            print("Tiles in the batch=",total_tiles)
+        corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
+        gt_ds_batch =            np.empty((total_tiles * tiles_in_sample, 2), dtype=float) 
+        target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ),  dtype=float) 
+        start_tile = 0
+        for nscene, scene_files in enumerate(ml_all_files):
+            for path in  scene_files:
+                '''
+                Create tiles list including neighbors
+                '''
+                full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
+                indx = 0;
+                for i, nt in enumerate(tiles[nscene]):
+                    ty = nt // width
+                    tx = nt % width
+                    for dy in range (-radius, radius+1):
+                        y = np.clip(ty+dy,0,height_m1)
+                        for dx in range (-radius, radius+1):
+                            x = np.clip(tx+dx,0,width_m1)
+                            full_tiles[indx] = y * width + x
+                            indx += 1
+                #now tile_list is np.array instead of the list, but it seems to be OK
+                img = ijt.imagej_tiff(path, corr_layers, tile_list=full_tiles) # tiles[nscene])
+                corr2d =           img.corr2d
+                target_disparity = img.target_disparity
+                gt_ds =            img.gt_ds
+                end_tile = start_tile + corr2d.shape[0]
+                 
+                if corr2d_batch is None:
+#                    corr2d_batch = np.empty((total_tiles, tiles_in_sample * len(corr_layers), corr2d.shape[-1]))
+                    corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d.shape[-1]))
+                gt_ds_batch            [start_tile:end_tile] = gt_ds
+                target_disparity_batch [start_tile:end_tile] = target_disparity
+                corr2d_batch           [start_tile:end_tile] = corr2d
+                start_tile = end_tile
+                """
+                 Sometimes get bad tile in ML file that was not bad in COMBO-DSI
+                 Need to recover
+                 np.argwhere(np.isnan(target_disparity_batch))                 
+                """
+        bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
+        if (len(bad_tiles)>0):
+            print ("*** Got %d bad tiles in a batch, replacing..."%(len(bad_tiles)), end=" ")
+            # for now - just repeat some good tile
+            for ibt in bad_tiles:
+                while np.isnan(target_disparity_batch[ibt]):
+                    irt = np.random.randint(0,total_tiles)
+                    if not np.isnan(target_disparity_batch[irt]):
+                        target_disparity_batch[ibt] = target_disparity_batch[irt]
+                        corr2d_batch[ibt] = corr2d_batch[irt]
+                        gt_ds_batch[ibt] = gt_ds_batch[irt]
+                        break
+            print (" done replacing")
+        self.corr2d_batch =           corr2d_batch
+        self.target_disparity_batch = target_disparity_batch
+        self.gt_ds_batch =            gt_ds_batch
+        return corr2d_batch, target_disparity_batch, gt_ds_batch
+
+
+
+    def writeTFRewcordsEpoch(self, tfr_filename, ml_list, files_list = None, set_ds= None, radius = 0): # test_set=False):
+#        train_filename = 'train.tfrecords'  # address to save the TFRecords file
+        # open the TFRecords file
+        if not  '.tfrecords' in tfr_filename:
+            tfr_filename += '.tfrecords'
+
+        tfr_filename=tfr_filename.replace(' ','_')
+        if files_list is None:
+            files_list = self.files_train
+            
+        if set_ds is None:
+            set_ds = self.train_ds
+        try:
+            os.makedirs(os.path.dirname(tfr_filename))
+            print("Created directory "+os.path.dirname(tfr_filename))
+        except:
+            print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
+            pass
+        #skip writing if file exists - it will be possible to continue or run several instances
+        if os.path.exists(tfr_filename):
+            print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
+            return     
+        writer = tf.python_io.TFRecordWriter(tfr_filename)
+#$        files_list = [self.files_train, self.files_test][test_set]
+        seed_list = np.arange(len(files_list))
+        np.random.shuffle(seed_list)
+        cluster_size = (2 * radius + 1) * (2 * radius + 1)
+        for nscene, seed_index in enumerate(seed_list):
+            corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
+            #shuffles tiles in a batch
+#            tiles_in_batch = len(target_disparity_batch)
+            tiles_in_batch =    corr2d_batch.shape[0]
+            clusters_in_batch = tiles_in_batch // cluster_size
+#            permut = np.random.permutation(tiles_in_batch)
+            permut = np.random.permutation(clusters_in_batch)
+            corr2d_clusters =           corr2d_batch.          reshape((clusters_in_batch,-1)) 
+            target_disparity_clusters = target_disparity_batch.reshape((clusters_in_batch,-1)) 
+            gt_ds_clusters =            gt_ds_batch.           reshape((clusters_in_batch,-1)) 
+                
+#            corr2d_batch_shuffled =           corr2d_batch[permut].reshape((corr2d_batch.shape[0], corr2d_batch.shape[1]*corr2d_batch.shape[2]))
+#            target_disparity_batch_shuffled = target_disparity_batch[permut].reshape((tiles_in_batch,1))
+#            gt_ds_batch_shuffled =            gt_ds_batch[permut]
+
+            corr2d_batch_shuffled =           corr2d_clusters[permut].          reshape((tiles_in_batch, -1))
+            target_disparity_batch_shuffled = target_disparity_clusters[permut].reshape((tiles_in_batch, -1))
+            gt_ds_batch_shuffled =            gt_ds_clusters[permut].           reshape((tiles_in_batch, -1))
+            
+            if nscene == 0:
+                dtype_feature_corr2d =   _dtype_feature(corr2d_batch_shuffled)
+                dtype_target_disparity = _dtype_feature(target_disparity_batch_shuffled)
+                dtype_feature_gt_ds =    _dtype_feature(gt_ds_batch_shuffled)
+
+            for i in range(tiles_in_batch):
+                x = corr2d_batch_shuffled[i].astype(np.float32)
+                y = target_disparity_batch_shuffled[i].astype(np.float32)
+                z = gt_ds_batch_shuffled[i].astype(np.float32)
+                d_feature = {'corr2d':          dtype_feature_corr2d(x),
+                             'target_disparity':dtype_target_disparity(y),
+                             'gt_ds':           dtype_feature_gt_ds(z)}
+                example = tf.train.Example(features=tf.train.Features(feature=d_feature))
+                writer.write(example.SerializeToString())
+            if (self.debug_level > 0):
+                print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        
+        writer.close()
+        sys.stdout.flush()        
+
+
+
+    
+    def showVariance(self,
+            rds_list,           # list of disparity/strength files, suchas training, testing 
+            disp_var_list,      # list of disparity variance files. Same shape(but last dim) as rds_list
+            num_neibs_list,    # list of number of tile neibs files. Same shape(but last dim) as rds_list
+            variance_min =       0.0,
+            variance_max =       1.5,
+            neibs_min =          9,
+            #Same parameters as for the histogram 
+#            disparity_bins =    1000,
+#            strength_bins =      100,
+#            disparity_min_drop =  -0.1,
+#            disparity_min_clip =  -0.1,
+#            disparity_max_drop = 100.0,
+#            disparity_max_clip = 100.0,
+#            strength_min_drop =    0.1,
+#            strength_min_clip =    0.1,
+#            strength_max_drop =    1.0,
+#            strength_max_clip =    0.9,
+            normalize =           False): # True):
+        good_tiles_list=[]
+        for nf, combo_rds in enumerate(rds_list):
+            disp_var =  disp_var_list[nf]
+            num_neibs = num_neibs_list[nf]
+            good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
+            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+                ds = combo_rds[ids]
+                disparity = ds[...,0]
+                strength =  ds[...,1]
+                variance =  disp_var[ids]
+                neibs =     num_neibs[ids]
+                good_tiles[ids] =  disparity >= self.disparity_min_drop
+                good_tiles[ids] &= disparity <= self.disparity_max_drop
+                good_tiles[ids] &= strength >=  self.strength_min_drop
+                good_tiles[ids] &= strength <=  self.strength_max_drop
+                good_tiles[ids] &= neibs    >=  neibs_min
+                good_tiles[ids] &= variance >=  variance_min
+                good_tiles[ids] &= variance <   variance_max
+                disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
+                strength =  np.nan_to_num(strength, copy = False)  # likely should never happen
+#                np.clip(disparity, self.disparity_min_clip, self.disparity_max_clip, out = disparity)
+#                np.clip(strength, self.strength_min_clip, self.strength_max_clip, out = strength)
+            good_tiles_list.append(good_tiles)
+        combo_rds = np.concatenate(rds_list)
+        hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
+            x =      combo_rds[...,1].flatten(),
+            y =      combo_rds[...,0].flatten(),
+            bins=    (self.strength_bins, self.disparity_bins),
+            range=   ((self.strength_min_clip,self.strength_max_clip),(self.disparity_min_clip,self.disparity_max_clip)),
+            normed=  normalize,
+            weights= np.concatenate(good_tiles_list).flatten())
+        
+        mytitle = "Disparity_Strength variance histogram"
+        fig = plt.figure()
+        fig.canvas.set_window_title(mytitle)
+        fig.suptitle("Min variance = %f, max variance = %f, min neibs = %d"%(variance_min, variance_max, neibs_min))
+#        plt.imshow(hist, vmin=0, vmax=.1 * hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+        plt.imshow(hist, vmin=0.0, vmax=300.0)#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+        plt.colorbar(orientation='horizontal') # location='bottom')
+        
+#        for i, combo_rds in enumerate(rds_list):
+#            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+#                combo_rds[ids][...,1]*= good_tiles_list[i][ids]
+#        return hist, xedges, yedges
+
+#MAIN
+if __name__ == "__main__":
+  try:
+      topdir_train = sys.argv[1]
+  except IndexError:
+#      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
+      topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
+  try:
+      topdir_test = sys.argv[2]
+  except IndexError:
+#      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
+      topdir_test =  "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
+      
+  try:
+      pathTFR =     sys.argv[3]
+  except IndexError:
+#      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
+      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #no trailing "/"
+
+  try:
+      ml_subdir =   sys.argv[4]
+  except IndexError:
+#      ml_subdir =   "ml"
+      ml_subdir =   "mlr32_18a"
+      
+      
+#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+  test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
+  #Parameters to generate neighbors data. Set radius to 0 to generate single-tile     
+  RADIUS = 2 # 5x5
+  MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
+  VARIANCE_THRESHOLD = 1.5
+  NUM_TRAIN_SETS = 8
+ 
+  if RADIUS == 0:
+    BATCH_DISP_BINS = 50 # 1000 * 1
+    BATCH_STR_BINS =  20 # 10
+  elif RADIUS == 1:
+    BATCH_DISP_BINS = 15 # 120 * 9
+    BATCH_STR_BINS =  8
+  else: # RADIUS = 2
+    BATCH_DISP_BINS = 10 # 40 * 25
+    BATCH_STR_BINS =  4
+
+  train_filenameTFR = pathTFR+"/train"        
+  test_filenameTFR =  pathTFR+"/test"
+#        disp_bins = 20,
+#      str_bins=10)
+
+#  corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(train_filenameTFR)
+#  print_time("Read %d tiles"%(corr2d.shape[0]))
+#  exit (0)
+  ex_data = ExploreData(
+               topdir_train =         topdir_train,
+               topdir_test =          topdir_test,
+               ml_subdir =            ml_subdir,
+               debug_level =          1, #3, ##0, #3,
+               disparity_bins =     200, #1000,
+               strength_bins =      100,
+               disparity_min_drop =  -0.1,
+               disparity_min_clip =  -0.1,
+               disparity_max_drop = 20.0, #100.0,
+               disparity_max_clip = 20.0, #100.0,
+               strength_min_drop =    0.1,
+               strength_min_clip =    0.1,
+               strength_max_drop =    1.0,
+               strength_max_clip =    0.9,
+               hist_sigma =           2.0,  # Blur log histogram
+               hist_cutoff=           0.001) #  of maximal  
+  
+  mytitle = "Disparity_Strength histogram"
+  fig = plt.figure()
+  fig.canvas.set_window_title(mytitle)
+  fig.suptitle(mytitle)
+#  plt.imshow(lhist,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  plt.imshow(ex_data.blurred_hist, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  plt.colorbar(orientation='horizontal') # location='bottom')
+  hist_to_batch = ex_data.assignBatchBins(
+      disp_bins = BATCH_DISP_BINS,
+      str_bins =  BATCH_STR_BINS)
+  bb_display = hist_to_batch.copy()
+  bb_display = ( 1+ (bb_display % 2) + 2 * ((bb_display % 20)//10)) * (hist_to_batch > 0) #).astype(float) 
+  fig2 = plt.figure()
+  fig2.canvas.set_window_title("Batch indices")
+  fig2.suptitle("Batch index for each disparity/strength cell")
+  plt.imshow(bb_display) #, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  
+  """ prepare test dataset """
+#  RADIUS = 1
+#  MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid
+#  VARIANCE_THRESHOLD = 1.5
+
+  if (RADIUS > 0):
+      disp_var_test,  num_neibs_test =  ex_data.exploreNeibs(ex_data.test_ds, RADIUS)
+      disp_var_train, num_neibs_train = ex_data.exploreNeibs(ex_data.train_ds, RADIUS)
+      
+      # show varinace histogram
+#      for var_thresh in [0.1, 1.0, 1.5, 2.0, 5.0]:
+      for var_thresh in [1.5]:
+           ex_data.showVariance(
+                rds_list =       [ex_data.train_ds, ex_data.test_ds],           # list of disparity/strength files, suchas training, testing 
+                disp_var_list =  [disp_var_train,  disp_var_test],      # list of disparity variance files. Same shape(but last dim) as rds_list
+                num_neibs_list = [num_neibs_train, num_neibs_test],    # list of number of tile neibs files. Same shape(but last dim) as rds_list
+                variance_min =       0.0,
+                variance_max =       var_thresh,
+                neibs_min =          MIN_NEIBS)
+           ex_data.showVariance(
+                rds_list =       [ex_data.train_ds, ex_data.test_ds],           # list of disparity/strength files, suchas training, testing 
+                disp_var_list =  [disp_var_train,  disp_var_test],      # list of disparity variance files. Same shape(but last dim) as rds_list
+                num_neibs_list = [num_neibs_train, num_neibs_test],    # list of number of tile neibs files. Same shape(but last dim) as rds_list
+                variance_min =       var_thresh,
+                variance_max =       1000.0,
+                neibs_min =          MIN_NEIBS)
+           pass
+      pass
+      
+  else:
+      disp_var_test,  num_neibs_test =  None, None    
+      disp_var_train, num_neibs_train = None, None    
+  
+  ml_list_train=ex_data.getMLList(ml_subdir, ex_data.files_train)
+  ml_list_test= ex_data.getMLList(ml_subdir, ex_data.files_test)
+
+  if RADIUS == 0 :
+      list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
+          data_ds =      ex_data.train_ds,
+          disp_var =     disp_var_train,      # difference between maximal and minimal disparity for each scene, each tile
+          disp_neibs =   num_neibs_train,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+          min_var =      0.0,                # Minimal tile variance to include
+          max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+          min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      pass
+#  ex_data.makeBatchLists(data_ds = ex_data.train_ds)
+      for train_var in range (NUM_TRAIN_SETS):
+          fpath =  train_filenameTFR+("%03d"%(train_var,))
+          ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds)
+          
+      list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
+          data_ds =      ex_data.test_ds,
+          disp_var =     disp_var_test,      # difference between maximal and minimal disparity for each scene, each tile
+          disp_neibs =   num_neibs_test,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+          min_var =      0.0,                # Minimal tile variance to include
+          max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+          min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      fpath =  test_filenameTFR # +("-%03d"%(train_var,))
+      ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_test, set_ds= ex_data.test_ds)
+      pass
+  else: # RADIUS > 0
+      # train
+      list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
+          data_ds =      ex_data.train_ds,
+          disp_var =     disp_var_train,      # difference between maximal and minimal disparity for each scene, each tile
+          disp_neibs =   num_neibs_train,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+          min_var =      0.0,                # Minimal tile variance to include
+          max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+          min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      num_le_train = num_batch_tiles_train.sum()
+      print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train))
+      for train_var in range (NUM_TRAIN_SETS):
+          fpath =  train_filenameTFR+("%03d_R%d_LE%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD))
+          ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
+
+      list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
+          data_ds =      ex_data.train_ds,
+          disp_var =     disp_var_train,      # difference between maximal and minimal disparity for each scene, each tile
+          disp_neibs =   num_neibs_train,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+          min_var =      VARIANCE_THRESHOLD,  # Minimal tile variance to include
+          max_var =      1000.0,              # Maximal tile variance to include
+          min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      num_gt_train = num_batch_tiles_train.sum()
+      high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train)
+      print("Number of > %f disparity variance tiles: %d, fraction = %f (train)"%(VARIANCE_THRESHOLD, num_gt_train, high_fract_train))
+      for train_var in range (NUM_TRAIN_SETS):
+          fpath =  (train_filenameTFR+("%03d_R%d_GT%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)))
+          ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
+          
+      # test
+      list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
+      data_ds =      ex_data.test_ds,
+      disp_var =     disp_var_test,      # difference between maximal and minimal disparity for each scene, each tile
+      disp_neibs =   num_neibs_test,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+      min_var =      0.0,                # Minimal tile variance to include
+      max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+      min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      num_le_test = num_batch_tiles_test.sum()
+      print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
+
+      fpath =  test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
+      ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
+
+      list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
+      data_ds =      ex_data.test_ds,
+      disp_var =     disp_var_test,      # difference between maximal and minimal disparity for each scene, each tile
+      disp_neibs =   num_neibs_test,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+      min_var =      VARIANCE_THRESHOLD, # Minimal tile variance to include
+      max_var =      1000.0,             # Maximal tile variance to include
+      min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      num_gt_test = num_batch_tiles_test.sum()
+      high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
+      print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test))
+      fpath =  test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
+      ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
+  plt.show()
+  scene = os.path.basename(test_corr)[:17]
+  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
+  fname =scene+'-'+scene_version 
+  img_filenameTFR = os.path.join(pathTFR,'img',fname)        
+  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
+  pass
+  
+  pass
+    

explore_data2.py

+#!/usr/bin/env python3
+from numpy import float64
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+import os
+import sys
+import glob
+import imagej_tiff as ijt
+import numpy as np
+import resource
+import timeit
+import matplotlib.pyplot as plt
+from scipy.ndimage.filters import gaussian_filter
+import time
+import tensorflow as tf
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+    
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end)
+    TIME_LAST = t
+
+def _dtype_feature(ndarray):
+    """match appropriate tf.train.Feature class with dtype of ndarray. """
+    assert isinstance(ndarray, np.ndarray)
+    dtype_ = ndarray.dtype
+    if dtype_ == np.float64 or dtype_ == np.float32:
+        return lambda array: tf.train.Feature(float_list=tf.train.FloatList(value=array))
+    elif dtype_ == np.int64:
+        return lambda array: tf.train.Feature(int64_list=tf.train.Int64List(value=array))
+    else:  
+        raise ValueError("The input should be numpy ndarray. \
+                           Instead got {}".format(ndarray.dtype))
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecordDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+    corr2d_list=[]
+    target_disparity_list=[]
+    gt_ds_list = []
+    for string_record in record_iterator:
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        corr2d_list.append(np.array(example.features.feature['corr2d'] .float_list .value))
+        target_disparity_list.append(np.array(example.features.feature['target_disparity'] .float_list .value[0]))
+        gt_ds_list.append(np.array(example.features.feature['gt_ds'] .float_list .value))
+    corr2d=            np.array(corr2d_list)
+    target_disparity = np.array(target_disparity_list)
+    gt_ds =            np.array(gt_ds_list)
+    return corr2d, target_disparity, gt_ds   
+
+def writeTFRewcordsImageTiles(img_path, tfr_filename): # test_set=False):
+#        train_filename = 'train.tfrecords'  # address to save the TFRecords file
+      # open the TFRecords file
+      num_tiles = 242*324 # fixme
+      all_image_tiles = np.array(range(num_tiles))
+      corr_layers =  ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
+      img =          ijt.imagej_tiff(test_corr, corr_layers, all_image_tiles)
+
+      corr2d =           img.corr2d.reshape((num_tiles,-1))
+      target_disparity = img.target_disparity.reshape((num_tiles,-1))
+      gt_ds =            img.gt_ds.reshape((num_tiles,-1))
+
+      if not  '.tfrecords' in tfr_filename:
+          tfr_filename += '.tfrecords'
+
+      tfr_filename=tfr_filename.replace(' ','_')
+      try:
+          os.makedirs(os.path.dirname(tfr_filename))
+      except:
+          pass     
+          
+      writer = tf.python_io.TFRecordWriter(tfr_filename)
+      dtype_feature_corr2d =   _dtype_feature(corr2d)
+      dtype_target_disparity = _dtype_feature(target_disparity)
+      dtype_feature_gt_ds =    _dtype_feature(gt_ds)
+      for i in range(num_tiles):
+          x = corr2d[i].astype(np.float32)
+          y = target_disparity[i].astype(np.float32)
+          z = gt_ds[i].astype(np.float32)
+          d_feature = {'corr2d':          dtype_feature_corr2d(x),
+                       'target_disparity':dtype_target_disparity(y),
+                       'gt_ds':           dtype_feature_gt_ds(z)}
+          example = tf.train.Example(features=tf.train.Features(feature=d_feature))
+          writer.write(example.SerializeToString())
+          pass
+      writer.close()
+      sys.stdout.flush()        
+
+
+
+class ExploreData:
+    PATTERN = "*-DSI_COMBO.tiff"
+#    ML_DIR = "ml"
+#    ML_PATTERN = "*-ML_DATA*OFFS*.tiff"
+    ML_PATTERN = "*-ML_DATA*MAIN*.tiff"
+#    ML_PATTERN = "*-ML_DATA*OFFS-0.20000_0.20000.tiff"
+    """
+1527182801_296892-ML_DATARND-32B-O-FZ0.05-OFFS-0.20000_0.20000.tiff
+    """
+    def getComboList(self, top_dir):
+#        patt = "*-DSI_COMBO.tiff"
+        tlist = []
+        for i in range(5):
+            pp = top_dir#) ,'**', patt) # works
+            for j in range (i):
+                pp = os.path.join(pp,'*')
+            pp = os.path.join(pp, ExploreData.PATTERN)
+            tlist += glob.glob(pp)
+            if (self.debug_level > 0):    
+                print (pp+" "+str(len(tlist)))
+        if (self.debug_level > 0):    
+            print("Found "+str(len(tlist))+" combo DSI files in "+top_dir+" :")
+            if (self.debug_level > 1):    
+                print("\n".join(tlist))
+        return tlist
+    
+    def loadComboFiles(self, tlist):
+        indx = 0
+        images = []
+        if (self.debug_level>2):
+            print(str(resource.getrusage(resource.RUSAGE_SELF)))
+        layers =  ['disparity_rig','strength_rig','disparity_main']
+        for combo_file in tlist:
+            tiff = ijt.imagej_tiff(combo_file,layers)
+            if not indx:
+                images = np.empty((len(tlist), tiff.image.shape[0],tiff.image.shape[1],tiff.image.shape[2]), tiff.image.dtype)
+            images[indx] = tiff.image
+            if (self.debug_level>2):
+                print(str(indx)+": "+str(resource.getrusage(resource.RUSAGE_SELF)))
+            indx += 1
+        return images
+    
+    def getHistogramDSI(
+            self, 
+            list_rds,
+            disparity_bins =    1000,
+            strength_bins =      100,
+            disparity_min_drop =  -0.1,
+            disparity_min_clip =  -0.1,
+            disparity_max_drop = 100.0,
+            disparity_max_clip = 100.0,
+            strength_min_drop =    0.1,
+            strength_min_clip =    0.1,
+            strength_max_drop =    1.0,
+            strength_max_clip =    0.9,
+            max_main_offset =      0.0,
+            normalize =           True,
+            no_histogram =        False            
+            ):
+        good_tiles_list=[]
+        for combo_rds in list_rds:
+            good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
+            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+                ds = combo_rds[ids]
+                disparity = ds[...,0]
+                strength =  ds[...,1]
+                good_tiles[ids] =  disparity >= disparity_min_drop
+                good_tiles[ids] &= disparity <= disparity_max_drop
+                good_tiles[ids] &= strength >=  strength_min_drop
+                good_tiles[ids] &= strength <=  strength_max_drop
+                if max_main_offset > 0.0:
+                    disparity_main = ds[...,2]
+                    good_tiles[ids] &= disparity_main <= (disparity + max_main_offset)
+                    good_tiles[ids] &= disparity_main >= (disparity - max_main_offset)
+                disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
+                strength =  np.nan_to_num(strength, copy = False)  # likely should never happen
+                np.clip(disparity, disparity_min_clip, disparity_max_clip, out = disparity)
+                np.clip(strength, strength_min_clip, strength_max_clip, out = strength)
+            good_tiles_list.append(good_tiles)
+        combo_rds = np.concatenate(list_rds)
+        hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
+            x =      combo_rds[...,1].flatten(),
+            y =      combo_rds[...,0].flatten(),
+            bins=    (strength_bins, disparity_bins),
+            range=   ((strength_min_clip,strength_max_clip),(disparity_min_clip,disparity_max_clip)),
+            normed=  normalize,
+            weights= np.concatenate(good_tiles_list).flatten())
+        for i, combo_rds in enumerate(list_rds):
+            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+                combo_rds[ids][...,1]*= good_tiles_list[i][ids]
+        return hist, xedges, yedges
+    
+    
+    def __init__(self,
+               topdir_train,
+               topdir_test,
+               ml_subdir,
+               max_main_offset =      2.0, # > 0.0 - do not use main camera tiles with offset more than this  
+               debug_level =          0,
+               disparity_bins =    1000,
+               strength_bins =      100,
+               disparity_min_drop =  -0.1,
+               disparity_min_clip =  -0.1,
+               disparity_max_drop = 100.0,
+               disparity_max_clip = 100.0,
+               strength_min_drop =    0.1,
+               strength_min_clip =    0.1,
+               strength_max_drop =    1.0,
+               strength_max_clip =    0.9,
+               
+               hist_sigma =           2.0,  # Blur log histogram
+               hist_cutoff=           0.001 #  of maximal  
+               ):
+    # file name
+        self.debug_level = debug_level
+        #self.testImageTiles()    
+        self.max_main_offset =    max_main_offset
+        self.disparity_bins =     disparity_bins
+        self.strength_bins =      strength_bins
+        self.disparity_min_drop = disparity_min_drop
+        self.disparity_min_clip = disparity_min_clip
+        self.disparity_max_drop = disparity_max_drop
+        self.disparity_max_clip = disparity_max_clip
+        self.strength_min_drop =  strength_min_drop
+        self.strength_min_clip =  strength_min_clip
+        self.strength_max_drop =  strength_max_drop
+        self.strength_max_clip =  strength_max_clip
+        self.hist_sigma =         hist_sigma # Blur log histogram
+        self.hist_cutoff=         hist_cutoff #  of maximal  
+        self.pre_log_offs =       0.001 # of histogram maximum
+        self.good_tiles =         None
+        self.files_train =        self.getComboList(topdir_train)
+        self.files_test =         self.getComboList(topdir_test)
+        
+        self.train_ds =           self.loadComboFiles(self.files_train)
+        self.test_ds =            self.loadComboFiles(self.files_test)
+        
+        self.num_tiles = self.train_ds.shape[1]*self.train_ds.shape[2] 
+        self.hist, xedges, yedges = self.getHistogramDSI(
+                list_rds =           [self.train_ds,self.test_ds], # combo_rds,
+                disparity_bins =     self.disparity_bins,
+                strength_bins =      self.strength_bins,
+                disparity_min_drop = self.disparity_min_drop,
+                disparity_min_clip = self.disparity_min_clip,
+                disparity_max_drop = self.disparity_max_drop,
+                disparity_max_clip = self.disparity_max_clip,
+                strength_min_drop =  self.strength_min_drop,
+                strength_min_clip =  self.strength_min_clip,
+                strength_max_drop =  self.strength_max_drop,
+                strength_max_clip =  self.strength_max_clip,
+                max_main_offset =    self.max_main_offset,
+                normalize =          True,
+                no_histogram =       False
+           )
+        log_offset = self.pre_log_offs * self.hist.max()
+        h_cutoff =   hist_cutoff * self.hist.max()
+        lhist =         np.log(self.hist + log_offset)
+        blurred_lhist = gaussian_filter(lhist, sigma = self.hist_sigma)
+        self.blurred_hist  = np.exp(blurred_lhist) - log_offset
+        self.good_tiles =  self.blurred_hist >= h_cutoff
+        self.blurred_hist *= self.good_tiles # set bad ones to zero 
+
+    def exploreNeibs(self,
+                     data_ds, # disparity/strength data for all files (train or test)
+                     radius,  # how far to look from center each side ( 1- 3x3, 2 - 5x5)
+                     disp_thesh = 5.0): # reduce effective variance for higher disparities
+        """
+        For each tile calculate difference between max and min among neighbors and number of qualifying neighbors (bad cewnter is not removed)
+        """
+        disp_min =   np.empty_like(data_ds[...,0], dtype = np.float)
+        disp_max =   np.empty_like(disp_min, dtype = np.float)
+        tile_neibs = np.zeros_like(disp_min, dtype = np.int)
+        dmin = data_ds[...,0].min()
+        dmax = data_ds[...,0].max()
+        good_tiles = self.getBB(data_ds) >= 0
+        side = 2 * radius + 1
+        for nf, ds in enumerate(data_ds):
+            disp = ds[...,0] 
+            height = disp.shape[0]
+            width = disp.shape[1]
+            bad_max = np.ones((height+side, width+side),  dtype=float) * dmax
+            bad_min = np.ones((height+side, width+side),  dtype=float) * dmin
+            good =    np.zeros((height+side, width+side), dtype=int)
+            #Assign centers of the array, replace bad tiles with max/min (so they will not change min/max) 
+            bad_max[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmax)
+            bad_min[radius:height+radius,radius:width+radius] = np.select([good_tiles[nf]],[disp],default = dmin)
+            good   [radius:height+radius,radius:width+radius] = good_tiles[nf]
+            disp_min  [nf,...] = disp 
+            disp_max  [nf,...] = disp
+            tile_neibs[nf,...] = good_tiles[nf]
+            for offset_y in range(-radius, radius+1):
+                oy = offset_y+radius
+                for offset_x in range(-radius, radius+1):
+                    ox = offset_x+radius
+                    if offset_y or offset_x: # Skip center - already copied
+                        np.minimum(disp_min[nf], bad_max[oy:oy+height, ox:ox+width], out=disp_min[nf])
+                        np.maximum(disp_max[nf], bad_min[oy:oy+height, ox:ox+width], out=disp_max[nf])
+                        tile_neibs[nf] +=  good[oy:oy+height, ox:ox+width]
+                        pass
+                    pass
+                pass
+            pass
+        
+        #disp_thesh
+        disp_avar = disp_max - disp_min
+        disp_rvar = disp_avar * disp_thesh / np.maximum(disp_max, 0.001) # removing division by 0 error - those tiles will be anyway discarded 
+        disp_var = np.select([disp_max >= disp_thesh, disp_max < disp_thesh],[disp_rvar,disp_avar])
+        return disp_var, tile_neibs
+
+    def assignBatchBins(self,
+                        disp_bins,
+                        str_bins,
+                        files_per_scene = 5,   # not used here, will be used when generating batches
+                        min_batch_choices=10,  # not used here, will be used when generating batches
+                        max_batch_files = 10): # not used here, will be used when generating batches
+        """
+        for each disparity/strength combination (self.disparity_bins * self.strength_bins = 1000*100) provide number of "large"
+        variable-size disparity/strength bin, or -1 if this disparity/strength combination does not seem right
+        """
+        self.files_per_scene = files_per_scene
+        self.min_batch_choices=min_batch_choices
+        self.max_batch_files = max_batch_files
+        
+        hist_to_batch =       np.zeros((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
+        hist_to_batch_multi = np.ones((self.blurred_hist.shape[0],self.blurred_hist.shape[1]),dtype=int) #zeros_like?
+        scale_hist= (disp_bins * str_bins)/self.blurred_hist.sum()
+        norm_b_hist =     self.blurred_hist * scale_hist
+        disp_list = [] # last disparity hist 
+#        disp_multi = [] # number of disp rows to fit
+        disp_run_tot = 0.0
+        disp_batch = 0
+        disp=0
+        num_batch_bins = disp_bins * str_bins
+        disp_hist = np.linspace(0, num_batch_bins, disp_bins+1)
+        batch_index = 0
+        num_members = np.zeros((num_batch_bins,),int)
+        while disp_batch < disp_bins:
+            #disp_multi.append(1)
+#        while (disp < self.disparity_bins):
+#            disp_target_tot =disp_hist[disp_batch+1]
+            disp_run_tot_new = disp_run_tot
+            disp0 = disp # start disaprity matching disp_run_tot 
+            while (disp_run_tot_new < disp_hist[disp_batch+1]) and (disp < self.disparity_bins):
+                disp_run_tot_new += norm_b_hist[:,disp].sum()
+                disp+=1;
+                disp_multi = 1
+                while   (disp_batch < (disp_bins - 1)) and (disp_run_tot_new >= disp_hist[disp_batch+2]):
+                    disp_batch += 1 # only if large disp_bins and very high hist value
+                    disp_multi += 1
+            # now  disp_run_tot - before this batch disparity col
+            str_bins_corr = str_bins * disp_multi # if too narrow disparity column - multiply number of strength columns
+            str_bins_corr_last = str_bins_corr -1
+            str_hist = np.linspace(disp_run_tot, disp_run_tot_new, str_bins_corr + 1)
+            str_run_tot_new = disp_run_tot
+#            str_batch = 0
+            str_index=0
+#            wide_col = norm_b_hist[:,disp0:disp] #disp0 - first column, disp - last+ 1
+            #iterate in linescan along the column
+            for si in range(self.strength_bins):
+                for di in range(disp0, disp,1):
+                    if norm_b_hist[si,di] > 0.0 :
+                        str_run_tot_new += norm_b_hist[si,di]
+                        # do not increment after last to avoid precision issues 
+                        if (batch_index < num_batch_bins) and (num_members[batch_index] > 0) and (str_index < str_bins_corr_last) and (str_run_tot_new > str_hist[str_index+1]):
+                            batch_index += 1
+                            str_index +=   1
+                        if batch_index < num_batch_bins :     
+                            hist_to_batch[si,di] = batch_index
+                            num_members[batch_index] += 1
+                        else:
+                            pass
+                    else:
+                        hist_to_batch[si,di] = -1
+                        
+            batch_index += 1 # it was not incremented afterthe last in the column to avoid rounding error 
+            disp_batch += 1
+            disp_run_tot = disp_run_tot_new
+            pass
+        self.hist_to_batch = hist_to_batch
+        return hist_to_batch        
+
+    def getBB(self, data_ds):
+        """
+        for each file, each tile get histogram index (or -1 for bad tiles)
+        """
+        hist_to_batch = self.hist_to_batch
+        files_batch_list = []
+        disp_step = ( self.disparity_max_clip - self.disparity_min_clip )/ self.disparity_bins 
+        str_step =  ( self.strength_max_clip -  self.strength_min_clip )/ self.strength_bins
+        bb = np.empty_like(data_ds[...,0],dtype=int)
+        for findx in range(data_ds.shape[0]):
+            ds = data_ds[findx]
+            gt = ds[...,1] > 0.0 # OK
+            db = (((ds[...,0] - self.disparity_min_clip)/disp_step).astype(int))*gt
+            sb = (((ds[...,1] - self.strength_min_clip)/ str_step).astype(int))*gt
+            np.clip(db, 0, self.disparity_bins-1, out = db)
+            np.clip(sb, 0, self.strength_bins-1, out = sb)
+            bb[findx] = (self.hist_to_batch[sb.reshape(self.num_tiles),db.reshape(self.num_tiles)])   .reshape(db.shape[0],db.shape[1]) + (gt -1)
+        return bb
+
+    def makeBatchLists(self,
+            data_ds =      None, # (disparity,strength) per scene, per tile
+            disp_var =     None, # difference between maximal and minimal disparity for each scene, each tile
+            disp_neibs =   None, # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+            min_var =      None, # Minimal tile variance to include
+            max_var =      None, # Maximal tile variance to include
+            min_neibs =    None):# Minimal number of valid tiles to include
+        if data_ds is None:
+             data_ds =      self.train_ds
+        hist_to_batch = self.hist_to_batch
+        num_batch_tiles = np.empty((data_ds.shape[0],self.hist_to_batch.max()+1),dtype = int) 
+        bb = self.getBB(data_ds)
+        use_neibs = not ((disp_var is None) or (disp_neibs is None) or (min_var is None) or (max_var is None) or (min_neibs is None))
+        list_of_file_lists=[]
+        for findx in range(data_ds.shape[0]):
+            foffs = findx * self.num_tiles 
+            lst = []
+            for i in range (self.hist_to_batch.max()+1):
+                lst.append([])
+#            bb1d = bb[findx].reshape(self.num_tiles)
+            if use_neibs:    
+                disp_var_tiles =   disp_var[findx].reshape(self.num_tiles)
+                disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
+            for n, indx in enumerate(bb[findx].reshape(self.num_tiles)):
+                if indx >= 0:
+                    if use_neibs:
+#                        disp_var_tiles =   disp_var[findx].reshape(self.num_tiles)
+#                        disp_neibs_tiles = disp_neibs[findx].reshape(self.num_tiles)
+                        if disp_neibs_tiles[n] < min_neibs:
+                            continue # too few neighbors
+                        if not disp_var_tiles[n] >= min_var:
+                            continue #too small variance 
+                        if not disp_var_tiles[n] <  max_var:
+                            continue #too large variance 
+                    lst[indx].append(foffs + n)
+            lst_arr=[]
+            for i,l in enumerate(lst):
+#                lst_arr.append(np.array(l,dtype = int))
+                lst_arr.append(l)
+                num_batch_tiles[findx,i] = len(l)
+            list_of_file_lists.append(lst_arr)
+        self.list_of_file_lists= list_of_file_lists
+        self.num_batch_tiles =   num_batch_tiles
+        return list_of_file_lists, num_batch_tiles
+    #todo: only use other files if there are no enough choices in the main file!
+    
+    
+    def augmentBatchFileIndices(self,
+                                 seed_index,
+                                 min_choices=None,
+                                 max_files = None,
+                                 set_ds = None
+                                 ):
+        if min_choices is None:
+            min_choices = self.min_batch_choices 
+        if max_files is None:
+            max_files =  self.max_batch_files
+        if set_ds is None:
+            set_ds = self.train_ds
+        full_num_choices = self.num_batch_tiles[seed_index].copy()
+        flist = [seed_index]
+        all_choices = list(range(self.num_batch_tiles.shape[0]))
+        all_choices.remove(seed_index)
+        for _ in range (max_files-1):
+            if full_num_choices.min() >= min_choices:
+                break
+            findx = np.random.choice(all_choices)
+            flist.append(findx)
+            all_choices.remove(findx)
+            full_num_choices += self.num_batch_tiles[findx]
+
+        file_tiles_sparse = [[] for _ in set_ds] #list of empty lists for each train scene (will be sparse)
+         
+        for nt in range(self.num_batch_tiles.shape[1]): #number of tiles per batch (not counting ml file variant) // radius2 - 40
+            tl = []
+            nchoices = 0
+            for findx in flist:
+                if (len(self.list_of_file_lists[findx][nt])):
+                    tl.append(self.list_of_file_lists[findx][nt])
+                nchoices+= self.num_batch_tiles[findx][nt]
+                if nchoices >= min_choices: # use minimum of extra files
+                    break;
+            while len(tl)==0:
+                print("** BUG! could not find a single candidate from files ",flist," for cell "+nt)
+                print("trying to use some other cell")
+                nt1 = np.random.randint(0,self.num_batch_tiles.shape[1])
+                for findx in flist:
+                    if (len(self.list_of_file_lists[findx][nt1])):
+                        tl.append(self.list_of_file_lists[findx][nt1])
+                    nchoices+= self.num_batch_tiles[findx][nt1]
+                    if nchoices >= min_choices: # use minimum of extra files
+                        break;
+            tile = np.random.choice(np.concatenate(tl))
+            """
+Traceback (most recent call last):
+  File "explore_data2.py", line 1041, in <module>
+    ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
+  File "explore_data2.py", line 761, in writeTFRewcordsEpoch
+    corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
+  File "explore_data2.py", line 556, in prepareBatchData
+    flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
+  File "explore_data2.py", line 494, in augmentBatchFileIndices
+    tile = np.random.choice(np.concatenate(tl))
+ValueError: need at least one array to concatenate
+            """
+#            print (nt, tile, tile//self.num_tiles, tile % self.num_tiles)
+            if not type (tile) is np.int64:
+                print("tile=",tile)
+            file_tiles_sparse[tile//self.num_tiles].append(tile % self.num_tiles)
+        file_tiles = []
+        for findx in flist:
+            file_tiles.append(np.sort(np.array(file_tiles_sparse[findx],dtype=int))) 
+        return flist, file_tiles # file indices, list if tile indices for each file   
+            
+            
+               
+                
+    def getMLList(self, ml_subdir, flist):
+        ml_list = []
+        for fn in flist:
+            ml_patt = os.path.join(os.path.dirname(fn), ml_subdir, ExploreData.ML_PATTERN)
+            ml_list.append(glob.glob(ml_patt))
+##        self.ml_list = ml_list
+        return ml_list
+            
+    def getBatchData(
+            self,
+            flist,
+            tiles,
+            ml_list,
+            ml_num = None ): # 0 - use all ml files for the scene, >0 select random number
+        if ml_num is None:
+            ml_num = self.files_per_scene
+        ml_all_files = []
+        for findx in flist:
+            mli =  list(range(len(ml_list[findx])))
+            if (ml_num > 0) and (ml_num < len(mli)):
+                mli_left = mli
+                mli = []
+                for _ in range(ml_num):
+                    ml = np.random.choice(mli_left)
+                    mli.append(ml)
+                    mli_left.remove(ml)
+            ml_files = []
+            for ml_index in mli:
+                ml_files.append(ml_list[findx][ml_index])
+            ml_all_files.append(ml_files)        
+                    
+        return ml_all_files
+    
+    def prepareBatchData(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
+        if min_choices is None:
+            min_choices = self.min_batch_choices
+        if max_files is None:
+            max_files = self.max_batch_files
+        if ml_num is None:
+            ml_num = self.files_per_scene
+        if set_ds is None:
+            set_ds = self.train_ds
+        tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
+        height = set_ds.shape[1]
+        width =  set_ds.shape[2]
+        width_m1 = width-1
+        height_m1 = height-1
+#        set_ds = [self.train_ds, self.test_ds][test_set]            
+        corr_layers =  ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
+        flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
+        
+#        ml_all_files = self.getBatchData(flist, tiles, ml_list,  ml_num) # 0 - use all ml files for the scene, >0 select random number
+        ml_all_files = self.getBatchData(flist, tiles, ml_list,  0) # ml_num) # 0 - use all ml files for the scene, >0 select random number
+        if self.debug_level > 1:
+            print ("==============",seed_index, flist)
+            for i, findx in enumerate(flist):
+                print(i,"\n".join(ml_all_files[i])) 
+                print(tiles[i]) 
+        total_tiles = 0
+        for i, t in enumerate(tiles):
+##          total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
+            total_tiles += len(t) # tiles per scene * offset files per scene
+        if self.debug_level > 1:
+            print("Tiles in the batch=",total_tiles)
+        corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
+        gt_ds_batch =            np.empty((total_tiles * tiles_in_sample, 2), dtype=float) 
+        target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ),  dtype=float) 
+        start_tile = 0
+        for nscene, scene_files in enumerate(ml_all_files):
+            '''
+            Create tiles list including neighbors
+            '''
+            full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
+            indx = 0;
+            for i, nt in enumerate(tiles[nscene]):
+                ty = nt // width
+                tx = nt % width
+                for dy in range (-radius, radius+1):
+                    y = np.clip(ty+dy,0,height_m1)
+                    for dx in range (-radius, radius+1):
+                        x = np.clip(tx+dx,0,width_m1)
+                        full_tiles[indx] = y * width + x
+                        indx += 1
+            """
+            Assign tiles to several correlation files
+            """
+            file_tiles = []
+            file_indices = []
+            for f in scene_files:
+                file_tiles.append([])
+            num_scene_files = len(scene_files)    
+            for t in full_tiles:
+                fi = np.random.randint(0, num_scene_files)
+                file_tiles[fi].append(t)
+                file_indices.append(fi)
+            corr2d_list =           []
+            target_disparity_list = []
+            gt_ds_list =            []
+            for fi, path in enumerate (scene_files):
+                img = ijt.imagej_tiff(path, corr_layers, tile_list=file_tiles[fi])
+                corr2d_list.append          (img.corr2d)
+                target_disparity_list.append(img.target_disparity)
+                gt_ds_list.append           (img.gt_ds)
+            img_indices = [0] * len(scene_files)
+            for i, fi in enumerate(file_indices):
+                ti = img_indices[fi]
+                img_indices[fi] += 1
+                if corr2d_batch is None:
+                    corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d_list[fi].shape[-1]))
+                gt_ds_batch            [start_tile] = gt_ds_list[fi][ti]
+                target_disparity_batch [start_tile] = target_disparity_list[fi][ti]
+                corr2d_batch           [start_tile] = corr2d_list[fi][ti]
+                start_tile +=  1
+            """
+             Sometimes get bad tile in ML file that was not bad in COMBO-DSI
+             Need to recover
+             np.argwhere(np.isnan(target_disparity_batch))                 
+            """
+        bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
+        if (len(bad_tiles)>0):
+            print ("*** Got %d bad tiles in a batch, no code to replace :-("%(len(bad_tiles)))
+            # for now - just repeat some good tile
+            """
+            for ibt in bad_tiles:
+                while np.isnan(target_disparity_batch[ibt]):
+                    irt = np.random.randint(0,total_tiles)
+                    if not np.isnan(target_disparity_batch[irt]):
+                        target_disparity_batch[ibt] = target_disparity_batch[irt]
+                        corr2d_batch[ibt] = corr2d_batch[irt]
+                        gt_ds_batch[ibt] = gt_ds_batch[irt]
+                        break
+            print (" done replacing")
+            """
+        self.corr2d_batch =           corr2d_batch
+        self.target_disparity_batch = target_disparity_batch
+        self.gt_ds_batch =            gt_ds_batch
+        return corr2d_batch, target_disparity_batch, gt_ds_batch
+
+    def prepareBatchDataOld(self, ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = None, radius = 0):
+        if min_choices is None:
+            min_choices = self.min_batch_choices
+        if max_files is None:
+            max_files = self.max_batch_files
+        if ml_num is None:
+            ml_num = self.files_per_scene
+        if set_ds is None:
+            set_ds = self.train_ds
+        tiles_in_sample = (2 * radius + 1) * (2 * radius + 1)
+        height = set_ds.shape[1]
+        width =  set_ds.shape[2]
+        width_m1 = width-1
+        height_m1 = height-1
+#        set_ds = [self.train_ds, self.test_ds][test_set]            
+        corr_layers =  ['hor-pairs', 'vert-pairs','diagm-pair', 'diago-pair']
+        flist,tiles = self.augmentBatchFileIndices(seed_index, min_choices, max_files, set_ds)
+        
+        ml_all_files = self.getBatchData(flist, tiles, ml_list,  ml_num) # 0 - use all ml files for the scene, >0 select random number
+        if self.debug_level > 1:
+            print ("==============",seed_index, flist)
+            for i, findx in enumerate(flist):
+                print(i,"\n".join(ml_all_files[i])) 
+                print(tiles[i]) 
+        total_tiles = 0
+        for i, t in enumerate(tiles):
+            total_tiles += len(t)*len(ml_all_files[i]) # tiles per scene * offset files per scene
+        if self.debug_level > 1:
+            print("Tiles in the batch=",total_tiles)
+        corr2d_batch = None # np.empty((total_tiles, len(corr_layers),81))
+        gt_ds_batch =            np.empty((total_tiles * tiles_in_sample, 2), dtype=float) 
+        target_disparity_batch = np.empty((total_tiles * tiles_in_sample, ),  dtype=float) 
+        start_tile = 0
+        for nscene, scene_files in enumerate(ml_all_files):
+            for path in  scene_files:
+                '''
+                Create tiles list including neighbors
+                '''
+                full_tiles = np.empty([len(tiles[nscene]) * tiles_in_sample], dtype = int)
+                indx = 0;
+                for i, nt in enumerate(tiles[nscene]):
+                    ty = nt // width
+                    tx = nt % width
+                    for dy in range (-radius, radius+1):
+                        y = np.clip(ty+dy,0,height_m1)
+                        for dx in range (-radius, radius+1):
+                            x = np.clip(tx+dx,0,width_m1)
+                            full_tiles[indx] = y * width + x
+                            indx += 1
+                #now tile_list is np.array instead of the list, but it seems to be OK
+                img = ijt.imagej_tiff(path, corr_layers, tile_list=full_tiles) # tiles[nscene])
+                corr2d =           img.corr2d
+                target_disparity = img.target_disparity
+                gt_ds =            img.gt_ds
+                end_tile = start_tile + corr2d.shape[0]
+                 
+                if corr2d_batch is None:
+#                    corr2d_batch = np.empty((total_tiles, tiles_in_sample * len(corr_layers), corr2d.shape[-1]))
+                    corr2d_batch = np.empty((total_tiles * tiles_in_sample, len(corr_layers), corr2d.shape[-1]))
+                gt_ds_batch            [start_tile:end_tile] = gt_ds
+                target_disparity_batch [start_tile:end_tile] = target_disparity
+                corr2d_batch           [start_tile:end_tile] = corr2d
+                start_tile = end_tile
+                """
+                 Sometimes get bad tile in ML file that was not bad in COMBO-DSI
+                 Need to recover
+                 np.argwhere(np.isnan(target_disparity_batch))                 
+                """
+        bad_tiles = np.argwhere(np.isnan(target_disparity_batch))
+        if (len(bad_tiles)>0):
+            print ("*** Got %d bad tiles in a batch, replacing..."%(len(bad_tiles)), end=" ")
+            # for now - just repeat some good tile
+            for ibt in bad_tiles:
+                while np.isnan(target_disparity_batch[ibt]):
+                    irt = np.random.randint(0,total_tiles)
+                    if not np.isnan(target_disparity_batch[irt]):
+                        target_disparity_batch[ibt] = target_disparity_batch[irt]
+                        corr2d_batch[ibt] = corr2d_batch[irt]
+                        gt_ds_batch[ibt] = gt_ds_batch[irt]
+                        break
+            print (" done replacing")
+        self.corr2d_batch =           corr2d_batch
+        self.target_disparity_batch = target_disparity_batch
+        self.gt_ds_batch =            gt_ds_batch
+        return corr2d_batch, target_disparity_batch, gt_ds_batch
+
+
+
+    def writeTFRewcordsEpoch(self, tfr_filename, ml_list, files_list = None, set_ds= None, radius = 0): # test_set=False):
+#        train_filename = 'train.tfrecords'  # address to save the TFRecords file
+        # open the TFRecords file
+        if not  '.tfrecords' in tfr_filename:
+            tfr_filename += '.tfrecords'
+
+        tfr_filename=tfr_filename.replace(' ','_')
+        if files_list is None:
+            files_list = self.files_train
+            
+        if set_ds is None:
+            set_ds = self.train_ds
+        try:
+            os.makedirs(os.path.dirname(tfr_filename))
+            print("Created directory "+os.path.dirname(tfr_filename))
+        except:
+            print("Directory "+os.path.dirname(tfr_filename)+" already exists, using it")
+            pass
+        #skip writing if file exists - it will be possible to continue or run several instances
+        if os.path.exists(tfr_filename):
+            print(tfr_filename+" already exists, skipping generation. Please remove and re-run this program if you want to regenerate the file")
+            return     
+        writer = tf.python_io.TFRecordWriter(tfr_filename)
+#$        files_list = [self.files_train, self.files_test][test_set]
+        seed_list = np.arange(len(files_list))
+        np.random.shuffle(seed_list)
+        cluster_size = (2 * radius + 1) * (2 * radius + 1)
+        for nscene, seed_index in enumerate(seed_list):
+            corr2d_batch, target_disparity_batch, gt_ds_batch = ex_data.prepareBatchData(ml_list, seed_index, min_choices=None, max_files = None, ml_num = None, set_ds = set_ds, radius = radius)
+            #shuffles tiles in a batch
+#            tiles_in_batch = len(target_disparity_batch)
+            tiles_in_batch =    corr2d_batch.shape[0]
+            clusters_in_batch = tiles_in_batch // cluster_size
+#            permut = np.random.permutation(tiles_in_batch)
+            permut = np.random.permutation(clusters_in_batch)
+            corr2d_clusters =           corr2d_batch.          reshape((clusters_in_batch,-1)) 
+            target_disparity_clusters = target_disparity_batch.reshape((clusters_in_batch,-1)) 
+            gt_ds_clusters =            gt_ds_batch.           reshape((clusters_in_batch,-1)) 
+                
+#            corr2d_batch_shuffled =           corr2d_batch[permut].reshape((corr2d_batch.shape[0], corr2d_batch.shape[1]*corr2d_batch.shape[2]))
+#            target_disparity_batch_shuffled = target_disparity_batch[permut].reshape((tiles_in_batch,1))
+#            gt_ds_batch_shuffled =            gt_ds_batch[permut]
+
+            corr2d_batch_shuffled =           corr2d_clusters[permut].          reshape((tiles_in_batch, -1))
+            target_disparity_batch_shuffled = target_disparity_clusters[permut].reshape((tiles_in_batch, -1))
+            gt_ds_batch_shuffled =            gt_ds_clusters[permut].           reshape((tiles_in_batch, -1))
+            
+            if nscene == 0:
+                dtype_feature_corr2d =   _dtype_feature(corr2d_batch_shuffled)
+                dtype_target_disparity = _dtype_feature(target_disparity_batch_shuffled)
+                dtype_feature_gt_ds =    _dtype_feature(gt_ds_batch_shuffled)
+
+            for i in range(tiles_in_batch):
+                x = corr2d_batch_shuffled[i].astype(np.float32)
+                y = target_disparity_batch_shuffled[i].astype(np.float32)
+                z = gt_ds_batch_shuffled[i].astype(np.float32)
+                d_feature = {'corr2d':          dtype_feature_corr2d(x),
+                             'target_disparity':dtype_target_disparity(y),
+                             'gt_ds':           dtype_feature_gt_ds(z)}
+                example = tf.train.Example(features=tf.train.Features(feature=d_feature))
+                writer.write(example.SerializeToString())
+            if (self.debug_level > 0):
+                print_time("Scene %d of %d -> %s"%(nscene, len(seed_list), tfr_filename))        
+        writer.close()
+        sys.stdout.flush()        
+
+
+
+    
+    def showVariance(self,
+            rds_list,           # list of disparity/strength files, suchas training, testing 
+            disp_var_list,      # list of disparity variance files. Same shape(but last dim) as rds_list
+            num_neibs_list,    # list of number of tile neibs files. Same shape(but last dim) as rds_list
+            variance_min =       0.0,
+            variance_max =       1.5,
+            neibs_min =          9,
+            #Same parameters as for the histogram 
+#            disparity_bins =    1000,
+#            strength_bins =      100,
+#            disparity_min_drop =  -0.1,
+#            disparity_min_clip =  -0.1,
+#            disparity_max_drop = 100.0,
+#            disparity_max_clip = 100.0,
+#            strength_min_drop =    0.1,
+#            strength_min_clip =    0.1,
+#            strength_max_drop =    1.0,
+#            strength_max_clip =    0.9,
+            normalize =           False): # True):
+        good_tiles_list=[]
+        for nf, combo_rds in enumerate(rds_list):
+            disp_var =  disp_var_list[nf]
+            num_neibs = num_neibs_list[nf]
+            good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
+            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+                ds = combo_rds[ids]
+                disparity = ds[...,0]
+                strength =  ds[...,1]
+                variance =  disp_var[ids]
+                neibs =     num_neibs[ids]
+                good_tiles[ids] =  disparity >= self.disparity_min_drop
+                good_tiles[ids] &= disparity <= self.disparity_max_drop
+                good_tiles[ids] &= strength >=  self.strength_min_drop
+                good_tiles[ids] &= strength <=  self.strength_max_drop
+                good_tiles[ids] &= neibs    >=  neibs_min
+                good_tiles[ids] &= variance >=  variance_min
+                good_tiles[ids] &= variance <   variance_max
+                disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
+                strength =  np.nan_to_num(strength, copy = False)  # likely should never happen
+#                np.clip(disparity, self.disparity_min_clip, self.disparity_max_clip, out = disparity)
+#                np.clip(strength, self.strength_min_clip, self.strength_max_clip, out = strength)
+            good_tiles_list.append(good_tiles)
+        combo_rds = np.concatenate(rds_list)
+        hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
+            x =      combo_rds[...,1].flatten(),
+            y =      combo_rds[...,0].flatten(),
+            bins=    (self.strength_bins, self.disparity_bins),
+            range=   ((self.strength_min_clip,self.strength_max_clip),(self.disparity_min_clip,self.disparity_max_clip)),
+            normed=  normalize,
+            weights= np.concatenate(good_tiles_list).flatten())
+        
+        mytitle = "Disparity_Strength variance histogram"
+        fig = plt.figure()
+        fig.canvas.set_window_title(mytitle)
+        fig.suptitle("Min variance = %f, max variance = %f, min neibs = %d"%(variance_min, variance_max, neibs_min))
+#        plt.imshow(hist, vmin=0, vmax=.1 * hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+        plt.imshow(hist, vmin=0.0, vmax=300.0)#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+        plt.colorbar(orientation='horizontal') # location='bottom')
+        
+#        for i, combo_rds in enumerate(rds_list):
+#            for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+#                combo_rds[ids][...,1]*= good_tiles_list[i][ids]
+#        return hist, xedges, yedges
+
+#MAIN
+if __name__ == "__main__":
+  try:
+      topdir_train = sys.argv[1]
+  except IndexError:
+#      topdir_train = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/train"#test" #all/"
+      topdir_train = "/home/eyesis/x3d_data/data_sets/train_mlr32_18a"
+  try:
+      topdir_test = sys.argv[2]
+  except IndexError:
+#      topdir_test = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/test"#test" #all/"
+      topdir_test =  "/home/eyesis/x3d_data/data_sets/test_mlr32_18a"
+      
+  try:
+      pathTFR =     sys.argv[3]
+  except IndexError:
+#      pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5" #no trailing "/"
+      pathTFR = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_4" #no trailing "/"
+
+  try:
+      ml_subdir =   sys.argv[4]
+  except IndexError:
+#      ml_subdir =   "ml"
+      ml_subdir =   "mlr32_18a"
+      
+      
+#  pathTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data_3x3b" #no trailing "/"
+#  test_corr = '/home/eyesis/x3d_data/models/var_main/www/html/x3domlet/models/all-clean/overlook/1527257933_150165/v04/mlr32_18a/1527257933_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
+  test_corr = '/home/eyesis/x3d_data/data_sets/test_mlr32_18a/1527256816_150165/v02/mlr32_18a/1527256816_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'
+#  test_corr = '/home/eyesis/x3d_data/models/dsi_combo_and_ml_all/state_street/1527256858_150165/v01/mlr32_18a/1527256858_150165-ML_DATA-32B-O-FZ0.05-MAIN.tiff'  
+  #Parameters to generate neighbors data. Set radius to 0 to generate single-tile     
+  RADIUS = 2 # 5x5
+  MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid == 9
+  VARIANCE_THRESHOLD = 1.5
+  NUM_TRAIN_SETS =     24 # 8
+ 
+  if RADIUS == 0:
+    BATCH_DISP_BINS = 50 # 1000 * 1
+    BATCH_STR_BINS =  20 # 10
+  elif RADIUS == 1:
+    BATCH_DISP_BINS = 15 # 120 * 9
+    BATCH_STR_BINS =  8
+  else: # RADIUS = 2
+    BATCH_DISP_BINS = 10 # 40 * 25
+    BATCH_STR_BINS =  4
+
+  train_filenameTFR = pathTFR+"/train"        
+  test_filenameTFR =  pathTFR+"/test"
+#        disp_bins = 20,
+#      str_bins=10)
+
+#  corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(train_filenameTFR)
+#  print_time("Read %d tiles"%(corr2d.shape[0]))
+#  exit (0)
+  ex_data = ExploreData(
+               topdir_train =         topdir_train,
+               topdir_test =          topdir_test,
+               ml_subdir =            ml_subdir,
+               debug_level =          1, #3, ##0, #3,
+               disparity_bins =     200, #1000,
+               strength_bins =      100,
+               disparity_min_drop =  -0.1,
+               disparity_min_clip =  -0.1,
+               disparity_max_drop = 20.0, #100.0,
+               disparity_max_clip = 20.0, #100.0,
+               strength_min_drop =    0.1,
+               strength_min_clip =    0.1,
+               strength_max_drop =    1.0,
+               strength_max_clip =    0.9,
+               hist_sigma =           2.0,  # Blur log histogram
+               hist_cutoff=           0.001) #  of maximal  
+  
+  mytitle = "Disparity_Strength histogram"
+  fig = plt.figure()
+  fig.canvas.set_window_title(mytitle)
+  fig.suptitle(mytitle)
+#  plt.imshow(lhist,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  plt.imshow(ex_data.blurred_hist, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  plt.colorbar(orientation='horizontal') # location='bottom')
+  hist_to_batch = ex_data.assignBatchBins(
+      disp_bins = BATCH_DISP_BINS,
+      str_bins =  BATCH_STR_BINS)
+  bb_display = hist_to_batch.copy()
+  bb_display = ( 1+ (bb_display % 2) + 2 * ((bb_display % 20)//10)) * (hist_to_batch > 0) #).astype(float) 
+  fig2 = plt.figure()
+  fig2.canvas.set_window_title("Batch indices")
+  fig2.suptitle("Batch index for each disparity/strength cell")
+  plt.imshow(bb_display) #, vmin=0, vmax=.1 * ex_data.blurred_hist.max())#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  
+  """ prepare test dataset """
+#  RADIUS = 1
+#  MIN_NEIBS = (2 * RADIUS + 1) * (2 * RADIUS + 1) # All tiles valid
+#  VARIANCE_THRESHOLD = 1.5
+
+  if (RADIUS > 0):
+      disp_var_test,  num_neibs_test =  ex_data.exploreNeibs(ex_data.test_ds, RADIUS)
+      disp_var_train, num_neibs_train = ex_data.exploreNeibs(ex_data.train_ds, RADIUS)
+      
+      # show varinace histogram
+#      for var_thresh in [0.1, 1.0, 1.5, 2.0, 5.0]:
+      for var_thresh in [1.5]:
+           ex_data.showVariance(
+                rds_list =       [ex_data.train_ds, ex_data.test_ds],           # list of disparity/strength files, suchas training, testing 
+                disp_var_list =  [disp_var_train,  disp_var_test],      # list of disparity variance files. Same shape(but last dim) as rds_list
+                num_neibs_list = [num_neibs_train, num_neibs_test],    # list of number of tile neibs files. Same shape(but last dim) as rds_list
+                variance_min =       0.0,
+                variance_max =       var_thresh,
+                neibs_min =          MIN_NEIBS)
+           ex_data.showVariance(
+                rds_list =       [ex_data.train_ds, ex_data.test_ds],           # list of disparity/strength files, suchas training, testing 
+                disp_var_list =  [disp_var_train,  disp_var_test],      # list of disparity variance files. Same shape(but last dim) as rds_list
+                num_neibs_list = [num_neibs_train, num_neibs_test],    # list of number of tile neibs files. Same shape(but last dim) as rds_list
+                variance_min =       var_thresh,
+                variance_max =       1000.0,
+                neibs_min =          MIN_NEIBS)
+           pass
+      pass
+      
+  else:
+      disp_var_test,  num_neibs_test =  None, None    
+      disp_var_train, num_neibs_train = None, None    
+  
+  ml_list_train=ex_data.getMLList(ml_subdir, ex_data.files_train)
+  ml_list_test= ex_data.getMLList(ml_subdir, ex_data.files_test)
+
+  if RADIUS == 0 :
+      list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
+          data_ds =      ex_data.train_ds,
+          disp_var =     disp_var_train,      # difference between maximal and minimal disparity for each scene, each tile
+          disp_neibs =   num_neibs_train,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+          min_var =      0.0,                # Minimal tile variance to include
+          max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+          min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      pass
+#  ex_data.makeBatchLists(data_ds = ex_data.train_ds)
+      for train_var in range (NUM_TRAIN_SETS):
+          fpath =  train_filenameTFR+("%03d"%(train_var,))
+          ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds)
+          
+      list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
+          data_ds =      ex_data.test_ds,
+          disp_var =     disp_var_test,      # difference between maximal and minimal disparity for each scene, each tile
+          disp_neibs =   num_neibs_test,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+          min_var =      0.0,                # Minimal tile variance to include
+          max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+          min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+      fpath =  test_filenameTFR # +("-%03d"%(train_var,))
+      ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_test, set_ds= ex_data.test_ds)
+      pass
+  else: # RADIUS > 0
+      # train
+      for train_var in range (NUM_TRAIN_SETS): # Recalculate list for each file - slower, but will alternate lvar/hvar
+          list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
+              data_ds =      ex_data.train_ds,
+              disp_var =     disp_var_train,      # difference between maximal and minimal disparity for each scene, each tile
+              disp_neibs =   num_neibs_train,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+              min_var =      0.0,                # Minimal tile variance to include
+              max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+              min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+          num_le_train = num_batch_tiles_train.sum()
+          print("Number of <= %f disparity variance tiles: %d (train)"%(VARIANCE_THRESHOLD, num_le_train))
+#      for train_var in range (NUM_TRAIN_SETS):
+          fpath =  train_filenameTFR+("%03d_R%d_LE%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD))
+          ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
+
+          list_of_file_lists_train, num_batch_tiles_train = ex_data.makeBatchLists( # results are also saved to self.*
+              data_ds =      ex_data.train_ds,
+              disp_var =     disp_var_train,      # difference between maximal and minimal disparity for each scene, each tile
+              disp_neibs =   num_neibs_train,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+              min_var =      VARIANCE_THRESHOLD,  # Minimal tile variance to include
+              max_var =      1000.0,              # Maximal tile variance to include
+              min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+          num_gt_train = num_batch_tiles_train.sum()
+          high_fract_train = 1.0 * num_gt_train / (num_le_train + num_gt_train)
+          print("Number of > %f disparity variance tiles: %d, fraction = %f (train)"%(VARIANCE_THRESHOLD, num_gt_train, high_fract_train))
+#      for train_var in range (NUM_TRAIN_SETS):
+          fpath =  (train_filenameTFR+("%03d_R%d_GT%4.1f"%(train_var,RADIUS,VARIANCE_THRESHOLD)))
+          ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_train, files_list = ex_data.files_train, set_ds= ex_data.train_ds, radius = RADIUS)
+
+          if train_var < 1: # make test files immediately after the train ones    
+          # test
+              list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
+              data_ds =      ex_data.test_ds,
+              disp_var =     disp_var_test,      # difference between maximal and minimal disparity for each scene, each tile
+              disp_neibs =   num_neibs_test,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+              min_var =      0.0,                # Minimal tile variance to include
+              max_var =      VARIANCE_THRESHOLD, # Maximal tile variance to include
+              min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+              num_le_test = num_batch_tiles_test.sum()
+              print("Number of <= %f disparity variance tiles: %d (est)"%(VARIANCE_THRESHOLD, num_le_test))
+        
+              fpath =  test_filenameTFR +("TEST_R%d_LE%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
+              ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
+        
+              list_of_file_lists_test, num_batch_tiles_test = ex_data.makeBatchLists( # results are also saved to self.*
+              data_ds =      ex_data.test_ds,
+              disp_var =     disp_var_test,      # difference between maximal and minimal disparity for each scene, each tile
+              disp_neibs =   num_neibs_test,     # number of valid tiles around each center tile (for 3x3 (radius = 1) - macximal is 9  
+              min_var =      VARIANCE_THRESHOLD, # Minimal tile variance to include
+              max_var =      1000.0,             # Maximal tile variance to include
+              min_neibs =    MIN_NEIBS)          # Minimal number of valid tiles to include
+              num_gt_test = num_batch_tiles_test.sum()
+              high_fract_test = 1.0 * num_gt_test / (num_le_test + num_gt_test)
+              print("Number of > %f disparity variance tiles: %d, fraction = %f (test)"%(VARIANCE_THRESHOLD, num_gt_test, high_fract_test))
+              fpath =  test_filenameTFR +("TEST_R%d_GT%4.1f"%(RADIUS,VARIANCE_THRESHOLD))
+              ex_data.writeTFRewcordsEpoch(fpath, ml_list = ml_list_test, files_list = ex_data.files_test, set_ds= ex_data.test_ds, radius = RADIUS)
+  plt.show()
+  scene = os.path.basename(test_corr)[:17]
+  scene_version= os.path.basename(os.path.dirname(os.path.dirname(test_corr)))
+  fname =scene+'-'+scene_version 
+  img_filenameTFR = os.path.join(pathTFR,'img',fname)
+  print_time("Saving test image %s as tiles..."%(img_filenameTFR),end = " ")        
+  writeTFRewcordsImageTiles(test_corr, img_filenameTFR)
+  print_time("Done")        
+  pass
+  
+  pass
+    

nn_ds_neibs2.py

@@ -31,7 +31,7 @@ FILES_PER_SCENE =    5 # number of random offset files for the scene to select f
 #MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
 #MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
 #MAX_EPOCH =        500
-LR =               1e-4 # learning rate
+LR =               1e-3 # learning rate
 LR100 =            1e-4
 USE_CONFIDENCE =     False
 ABSOLUTE_DISPARITY = False # True # False # True # False
@@ -47,8 +47,8 @@ RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batche
 #BATCH_SIZE =       1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
 BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
 SHUFFLE_EPOCH =    True
-NET_ARCH1 =          0 # 6 #0 # 4 # 3 # overwrite with argv?
-NET_ARCH2 =          0 # 6 # 0 # 3 # overwrite with argv?
+NET_ARCH1 =          0 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =          3 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
 ONLY_TILE =          None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
 ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
 
@@ -331,7 +331,7 @@ except IndexError:
 train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
 """
 files_img =        ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords']
-result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R.npy'
+result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-3.npy'
 
 files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords",
                     "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords",
@@ -364,11 +364,11 @@ import tensorflow as tf
 import tensorflow.contrib.slim as slim
 
 
-#try:
-eval_results(result_file, ABSOLUTE_DISPARITY)
-exit(0)
-#except:
-#    pass
+try:
+    eval_results(result_file, ABSOLUTE_DISPARITY)
+#exit(0)
+except:
+    pass
 
 datasets_img = []
 for fpath in files_img:
@@ -503,7 +503,7 @@ dataset_img_size //= BATCH_SIZE
 
 #print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
 dataset_tt = dataset_tt.batch(BATCH_SIZE)
-#dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
 iterator_tt = dataset_tt.make_initializable_iterator()
 next_element_tt = iterator_tt.get_next()
 #print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))

nn_ds_neibs3.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            1e-4
+LR200 =            3e-5
+LR400 =            1e-5
+LR600 =            3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     1000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =          8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =          3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =         True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+
+#DEBUG_PACK_TILES = True
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     1 # 1 - 3x3, 2 - 5x5 tiles
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+    #        target_disparity_list.append(np.array(example.features.feature['target_disparity'].float_list.value[0], dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'], copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'], copy = False)
+
+
+
+def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+
+    datasets_data = []
+#    for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
+    for nrec in range(len(datasets_lvar_data)):
+        rec1 = datasets_lvar_data[nrec]
+        rec2 = datasets_hvar_data[nrec]
+        
+        rec = {'corr2d':           np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
+               'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
+               'gt_ds':            np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
+
+        rec['corr2d'][0::2] =           rec1['corr2d'] 
+        rec['corr2d'][1::2] =           rec2['corr2d'] 
+        
+        rec['target_disparity'][0::2] = rec1['target_disparity'] 
+        rec['target_disparity'][1::2] = rec2['target_disparity'] 
+
+        rec['gt_ds'][0::2] =            rec1['gt_ds'] 
+        rec['gt_ds'][1::2] =            rec2['gt_ds']
+#        if del_src:
+#            rec1['corr2d'] = None
+#            rec1['target_disparity'] = None
+#            rec1['gt_ds'] = None
+#            rec2['corr2d'] = None
+#            rec2['target_disparity'] = None
+#            rec2['gt_ds'] = None
+        if del_src:
+            datasets_lvar_data[nrec] = None
+            datasets_hvar_data[nrec] = None
+        datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =    -0.1, #minimal GT disparity
+                 max_disp =    20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =      2.0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+files_img =        ['/home/eyesis/x3d_data/data_sets/tf_data_3x3b/img/1527257933_150165-v04.tfrecords']
+#result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-4.npy'
+result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04'+SUFFIX+'.npy'
+
+files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train002_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train003_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train004_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train005_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train006_R1_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train007_R1_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["/home/eyesis/x3d_data/data_sets/tf_data_rand2/train000_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train001_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train002_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train003_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train004_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train005_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train006_R1_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_rand2/train007_R1_GT_1.5.tfrecords",
+]
+#files_train_hvar = []
+#file_test_lvar=     "/home/eyesis/x3d_data/data_sets/tf_data_3x3a/train000_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
+file_test_lvar=     "/home/eyesis/x3d_data/data_sets/tf_data_rand2/testTEST_R1_LE_1.5.tfrecords"
+file_test_hvar=     "/home/eyesis/x3d_data/data_sets/tf_data_rand2/testTEST_R1_GT_1.5.tfrecords" # None # "/home/eyesis/x3d_data/data_sets/tf_data_3x3/train-002_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
+#file_test_hvar=  None
+weight_hvar = 0.13
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+try:
+    eval_results(result_file, ABSOLUTE_DISPARITY)
+#exit(0)
+except:
+    pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+datasets_train_hvar = []
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+if (file_test_lvar):
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
+    dataset_test_lvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+if (file_test_hvar):
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
+    dataset_test_hvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
+    if (file_test_lvar):
+        reduce_tile_size([dataset_test_lvar], TILE_LAYERS, TILE_SIDE)
+    if (file_test_hvar):
+        reduce_tile_size([dataset_test_hvar], TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+if (file_test_lvar):
+    print_time("Reshaping test data (low variance)", end="")
+    reformat_to_clusters([dataset_test_lvar])
+    print_time("  Done")
+if (file_test_hvar):    
+    print_time("Reshaping test data (high variance)", end="")
+    reformat_to_clusters([dataset_test_hvar])
+    print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+
+if ZIP_LHVAR:
+    datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar)
+    pass
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+if (file_test_lvar):
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+if (file_test_hvar):
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(dataset_test_lvar['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+
+
+"""
+iterator_test =  dataset_test.make_initializable_iterator()
+next_element_test =  iterator_test.get_next()
+"""
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i),     reuse = reuse | (j > 0)))
+                   if num_non_sum > 0:
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))    
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           
+    return fc[-1]
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?:9,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+                inter_list.append(network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8))
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2)   
+#corr2d9x325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[None,cluster_size,FEATURES_PER_TILE]) , tf.reshape(next_element_tt['target_disparity'], [None,cluster_size, 1])],2)
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+    
+#In GPU - reformat inputs
+
+##corr2d325 = tf.concat([next_element_tt['corr2d'], next_element_tt['target_disparity']],1)
+
+#Should have shape (?,9,325)
+corr2d9x325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE]) , tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1])],2)
+
+#corr2d9x324 = tf.reshape( next_element_tt['corr2d'],          [-1, cluster_size, FEATURES_PER_TILE], name = 'corr2d9x324')
+#td9x1 =       tf.reshape(next_element_tt['target_disparity'], [-1, cluster_size, 1],    name = 'td9x1')
+#corr2d9x325 = tf.concat([corr2d9x324 , td9x1],2, name = 'corr2d9x325')
+
+
+#    in_features = tf.concat([corr2d,target_disparity],0)
+
+#out =       network_fc_simple(input=corr2d325, arch = NET_ARCH1)
+out =       network_siam(input=corr2d9x325,
+                         layout1 =   NN_LAYOUT1, 
+                         layout2 =   NN_LAYOUT2,
+                         sym8 =      SYM8_SUB,
+                         only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+              
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
+    tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",   tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",   tf_img_test0)
+    tf.summary.scalar("img_test9",   tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs3_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    loss_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_avg = 0.0     
+    train2_avg = 0.0
+    test_avg = 0.0     
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+#    num_train_variants = len(files_train_lvar)
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        learning_rate = [LR,LR100][epoch >=100] 
+
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        
+#        disp_out=  np.empty((dataset_train_size * BATCH_SIZE), dtype=np.float32)
+        for i in range(dataset_train_size):
+            try:
+#                train_summary,_, G_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, corr2d325_out  = sess.run(
+                train_summary,_, G_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        G_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+#                        corr2d325,
+                    ],
+                    feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                # save all for now as a test
+                #train_writer.add_summary(summary, i)
+                #train_writer.add_summary(train_summary, i)
+#                disp_out[BATCH_SIZE*i:BATCH_SIZE*(i+1)] = output.flatten()
+                loss_train_hist[i] =  G_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+        train_avg =       np.average(loss_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            
+            for i in range(dataset_test_size):
+                try:
+#                    test_summary, G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                    test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         G_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                            feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    loss_test_hist[i] =  G_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+                test_avg =  np.average(loss_test_hist).astype(np.float32)     
+                test2_avg = np.average(loss2_test_hist).astype(np.float32)
+                gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+#        _,_=sess.run([tf_ph_G_loss,tf_ph_sq_diff],feed_dict={tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg})
+        
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f (%f %f) dbg:%f %f"%(epoch,i,train_avg, test_avg,train2_avg, test2_avg, gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+        #        start_offs=0
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                                feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+        #                loss_test_hist[i] =  G_loss_tested
+        #                loss2_test_hist[i] = out_cost1
+        #                gtvar_test_hist[i] = gt_variance
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+        #                disp_out[BATCH_SIZE*i:BATCH_SIZE*(i+1)] = output.flatten()
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+                
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY)
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)

nn_ds_neibs4.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+import sys
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            1e-4
+LR200 =            3e-5
+LR400 =            1e-5
+LR600 =            3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     1000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+#EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+BATCH_SIZE =       2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =          0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =          0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =         False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+
+#DEBUG_PACK_TILES = True
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     2 # 1 # 1 - 3x3, 2 - 5x5 tiles
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'],           copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'],            copy = False)
+
+
+
+def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+
+    datasets_data = []
+#    for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
+    for nrec in range(len(datasets_lvar_data)):
+        rec1 = datasets_lvar_data[nrec]
+        rec2 = datasets_hvar_data[nrec]
+        
+        rec = {'corr2d':           np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
+               'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
+               'gt_ds':            np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
+
+        rec['corr2d'][0::2] =           rec1['corr2d'] 
+        rec['corr2d'][1::2] =           rec2['corr2d'] 
+        
+        rec['target_disparity'][0::2] = rec1['target_disparity'] 
+        rec['target_disparity'][1::2] = rec2['target_disparity'] 
+
+        rec['gt_ds'][0::2] =            rec1['gt_ds'] 
+        rec['gt_ds'][1::2] =            rec2['gt_ds']
+#        if del_src:
+#            rec1['corr2d'] = None
+#            rec1['target_disparity'] = None
+#            rec1['gt_ds'] = None
+#            rec2['corr2d'] = None
+#            rec2['target_disparity'] = None
+#            rec2['gt_ds'] = None
+        if del_src:
+            datasets_lvar_data[nrec] = None
+            datasets_hvar_data[nrec] = None
+        datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =    -0.1, #minimal GT disparity
+                 max_disp =    20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =      2.0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+files_img =        ['/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/img/1527257933_150165-v04.tfrecords']
+#result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-4.npy'
+result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/rslt/1527257933_150165-v04'+SUFFIX+'.npy'
+
+files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train002_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train003_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train004_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train005_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train006_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords", # replace with GT when ready
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
+]
+#files_train_hvar = []
+#file_test_lvar=     "/home/eyesis/x3d_data/data_sets/tf_data_3x3a/train000_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
+#replace when available
+file_test_lvar=     "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_LE_1.5.tfrecords"
+file_test_hvar=     "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords" # None # "/home/eyesis/x3d_data/data_sets/tf_data_3x3/train-002_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
+#file_test_hvar=  None
+weight_hvar = 0.13
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+try:
+    eval_results(result_file, ABSOLUTE_DISPARITY)
+#exit(0)
+except:
+    pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":           corr2d,
+                                "target_disparity": target_disparity,
+                                "gt_ds":            gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+datasets_train_hvar = []
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+if (file_test_lvar):
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
+    dataset_test_lvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+if (file_test_hvar):
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
+    dataset_test_hvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
+    if (file_test_lvar):
+        reduce_tile_size([dataset_test_lvar], TILE_LAYERS, TILE_SIDE)
+    if (file_test_hvar):
+        reduce_tile_size([dataset_test_hvar], TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+if (file_test_lvar):
+    print_time("Reshaping test data (low variance)", end="")
+    reformat_to_clusters([dataset_test_lvar])
+    print_time("  Done")
+if (file_test_hvar):    
+    print_time("Reshaping test data (high variance)", end="")
+    reformat_to_clusters([dataset_test_hvar])
+    print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+
+if ZIP_LHVAR:
+    print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
+    datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar)
+    print_time("  Done")
+    pass
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+if (file_test_lvar):
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+if (file_test_hvar):
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+#dataset_tt    TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>    
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(dataset_test_lvar['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
+"""
+next_element_tt    dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}    
+    'corr2d' (140405473715624)    Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)    
+    'gt_ds' (140405473715680)    Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)    
+    'target_disparity' (140405501995888)    Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)    
+"""
+
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i),     reuse = reuse | (j > 0)))
+                   if num_non_sum > 0:
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))    
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           
+    return fc[-1]
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?,9,325]-> [?,25,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+                inter_list.append(network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8))
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2)   
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+    
+corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
+                          tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
+
+out =       network_siam(input=corr2d_Nx325,
+                         layout1 =   NN_LAYOUT1, 
+                         layout2 =   NN_LAYOUT2,
+                         sym8 =      SYM8_SUB,
+                         only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+              
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
+    tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",   tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",   tf_img_test0)
+    tf.summary.scalar("img_test9",   tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs4_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    loss_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_avg = 0.0     
+    train2_avg = 0.0
+    test_avg = 0.0     
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+#    num_train_variants = len(files_train_lvar)
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        print ("sr1",file=sys.stderr,end=" ")
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        for i in range(dataset_train_size):
+            try:
+                train_summary,_, G_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        G_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+                    ],
+                    feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                loss_train_hist[i] =  G_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+
+        train_avg =       np.average(loss_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            tst_avg = [0.0]*dataset_test_size
+            tst2_avg = [0.0]*dataset_test_size
+            for i in range(dataset_test_size):
+                try:
+                    test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         G_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                            feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    loss_test_hist[i] =  G_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+                test_avg =  np.average(loss_test_hist).astype(np.float32)
+                tst_avg[i] =  test_avg   
+                test2_avg = np.average(loss2_test_hist).astype(np.float32)
+                tst2_avg[i] =  test2_avg   
+                gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                                feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY)
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)

nn_ds_neibs5.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+import sys
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            1e-4
+LR200 =            1e-4 # 3e-5
+LR400 =            1e-4 # 1e-5
+LR600 =            1e-4 # 3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     1000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+#EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+BATCH_SIZE =       2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =         0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =         3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =         False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+
+#DEBUG_PACK_TILES = True
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     2 # 1 # 1 - 3x3, 2 - 5x5 tiles
+SHUFFLE_FILES  =     True
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'],           copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'],            copy = False)
+
+
+
+def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+
+    datasets_data = []
+    permut_l = np.random.permutation(len(datasets_lvar_data))    
+    permut_h = np.random.permutation(len(datasets_hvar_data))
+    """
+    TODO: add shuffling batches in each record
+    """    
+#    for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
+    for nrec in range(len(datasets_lvar_data)):
+        if shuffle:
+            rec1 = datasets_lvar_data[permut_l[nrec]]
+            rec2 = datasets_hvar_data[permut_h[nrec]]
+        else:
+            rec1 = datasets_lvar_data[nrec]
+            rec2 = datasets_hvar_data[nrec]
+            
+#        print ("\n",nrec, permut_l[nrec], permut_h[nrec])
+#        print (rec1['corr2d'].shape,rec2['corr2d'].shape)
+        rec = {'corr2d':           np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
+               'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
+               'gt_ds':            np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
+
+        rec['corr2d'][0::2] =           rec1['corr2d'] 
+        rec['corr2d'][1::2] =           rec2['corr2d'] 
+        
+        rec['target_disparity'][0::2] = rec1['target_disparity'] 
+        rec['target_disparity'][1::2] = rec2['target_disparity'] 
+
+        rec['gt_ds'][0::2] =            rec1['gt_ds'] 
+        rec['gt_ds'][1::2] =            rec2['gt_ds']
+        if del_src:
+            datasets_lvar_data[nrec] = None
+            datasets_hvar_data[nrec] = None
+        datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =       -0.1, #minimal GT disparity
+                 max_disp =       20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =         2.0,
+                 radius =          0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool) 
+    not_nan_ext[radius:-radius,radius:-radius] = not_nan
+    for dy in range(2*radius+1):
+        for dx in range(2*radius+1):
+            not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
+    not_nan = not_nan_ext[radius:-radius,radius:-radius]         
+        
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+files_img =        ['/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/img/1527257933_150165-v04.tfrecords']
+#result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_3x3b/rslt/1527257933_150165-v04R-M0-4.npy'
+result_file =       '/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/rslt/1527257933_150165-v04'+SUFFIX+'.npy'
+
+files_train_lvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train002_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train003_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train004_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train005_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train006_R2_LE_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords", # replace with GT when ready
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train001_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train002_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train003_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train004_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train005_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train006_R2_GT_1.5.tfrecords",
+                    "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/train000_R2_GT_1.5.tfrecords",
+]
+#files_train_hvar = []
+#file_test_lvar=     "/home/eyesis/x3d_data/data_sets/tf_data_3x3a/train000_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
+#replace when available
+file_test_lvar=     "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/testTEST_R2_LE_1.5.tfrecords" #train000_R2_LE_1.5.tfrecords"
+file_test_hvar=     "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg/testTEST_R2_GT_1.5.tfrecords" #train002_R2_GT_1.5.tfrecords" # None # "/home/eyesis/x3d_data/data_sets/tf_data_3x3/train-002_R1_LE_1.5.tfrecords" # "/home/eyesis/x3d_data/data_sets/train-000_R1_LE_1.5.tfrecords"
+#file_test_hvar=  None
+weight_hvar = 0.13
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+try:
+    eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
+#exit(0)
+except:
+    pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":           corr2d,
+                                "target_disparity": target_disparity,
+                                "gt_ds":            gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+datasets_train_hvar = []
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+if (file_test_lvar):
+    print_time("Importing test data (low variance) from "+file_test_lvar, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
+    dataset_test_lvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+if (file_test_hvar):
+    print_time("Importing test data (high variance) from "+file_test_hvar, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
+    dataset_test_hvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
+    if (file_test_lvar):
+        reduce_tile_size([dataset_test_lvar], TILE_LAYERS, TILE_SIDE)
+    if (file_test_hvar):
+        reduce_tile_size([dataset_test_hvar], TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+if (file_test_lvar):
+    print_time("Reshaping test data (low variance)", end="")
+    reformat_to_clusters([dataset_test_lvar])
+    print_time("  Done")
+if (file_test_hvar):    
+    print_time("Reshaping test data (high variance)", end="")
+    reformat_to_clusters([dataset_test_hvar])
+    print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+
+if ZIP_LHVAR:
+    print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
+    datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
+    print_time("  Done")
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+if (file_test_lvar):
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+if (file_test_hvar):
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+#dataset_tt    TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>    
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(dataset_test_lvar['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
+"""
+next_element_tt    dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}    
+    'corr2d' (140405473715624)    Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)    
+    'gt_ds' (140405473715680)    Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)    
+    'target_disparity' (140405501995888)    Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)    
+"""
+
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i),     reuse = reuse | (j > 0)))
+                   if num_non_sum > 0:
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))    
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           
+    return fc[-1]
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?,9,325]-> [?,25,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+                inter_list.append(network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8))
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2)   
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+    
+corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
+                          tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
+
+out =       network_siam(input=corr2d_Nx325,
+                         layout1 =   NN_LAYOUT1, 
+                         layout2 =   NN_LAYOUT2,
+                         sym8 =      SYM8_SUB,
+                         only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+              
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
+    tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",   tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",   tf_img_test0)
+    tf.summary.scalar("img_test9",   tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs5_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    loss_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_avg = 0.0     
+    train2_avg = 0.0
+    test_avg = 0.0     
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+#    num_train_variants = len(files_train_lvar)
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        print ("sr1",file=sys.stderr,end=" ")
+        if (file_index == 0) and SHUFFLE_FILES:
+            print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
+            datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
+            print_time("  Done")
+            
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        for i in range(dataset_train_size):
+            try:
+                train_summary,_, G_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        G_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+                    ],
+                    feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                loss_train_hist[i] =  G_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+
+        train_avg =       np.average(loss_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        tst_avg =        [0.0]*len(datasets_test)
+        tst2_avg =       [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            for i in range(dataset_test_size):
+                try:
+                    test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         G_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                            feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    loss_test_hist[i] =  G_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+            test_avg =  np.average(loss_test_hist).astype(np.float32)
+            tst_avg[ntest] =  test_avg   
+            test2_avg = np.average(loss2_test_hist).astype(np.float32)
+            tst2_avg[ntest] =  test2_avg   
+            gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                                feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)                
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)

nn_ds_neibs6.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+import sys
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            LR    # 1e-4
+LR200 =            LR100 # 3e-5
+LR400 =            LR200 # 1e-5
+LR600 =            LR400 # 3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     3000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+#EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+BATCH_SIZE =       2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =        0 #3 # 0 #  0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =        0 # 0 # 3 #  0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =         False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+
+#DEBUG_PACK_TILES = True
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     2 # 1 # 1 - 3x3, 2 - 5x5 tiles
+SHUFFLE_FILES  =     True
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'],           copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'],            copy = False)
+
+def permute_to_swaps(perm):
+    pairs = []
+    for i in range(len(perm)):
+        w = np.where(perm == i)[0][0]
+        if w != i:
+            pairs.append([i,w])
+            perm[w] = perm[i]
+            perm[i] = i
+    return pairs        
+        
+def shuffle_in_place(datasets_data, indx, period):
+    swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
+    num_entries = datasets_data[0]['corr2d'].shape[0] // period
+    for swp in swaps:
+        ds0 = datasets_data[swp[0]]
+        ds1 = datasets_data[swp[1]]
+        tmp = ds0['corr2d'][indx::period].copy() 
+        ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
+        ds1['corr2d'][indx::period] = tmp
+
+        tmp = ds0['target_disparity'][indx::period].copy() 
+        ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
+        ds1['target_disparity'][indx::period] = tmp
+
+        tmp = ds0['gt_ds'][indx::period].copy() 
+        ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
+        ds1['gt_ds'][indx::period] = tmp
+    
+    
+    
+def zip_lvar_hvar(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+
+    datasets_data = []
+    permut_l = np.random.permutation(len(datasets_lvar_data))    
+    permut_h = np.random.permutation(len(datasets_hvar_data))
+    """
+    TODO: add shuffling batches in each record
+    """    
+#    for rec1, rec2 in zip(datasets_lvar_data, datasets_hvar_data):
+    for nrec in range(len(datasets_lvar_data)):
+        if shuffle:
+            rec1 = datasets_lvar_data[permut_l[nrec]]
+            rec2 = datasets_hvar_data[permut_h[nrec]]
+        else:
+            rec1 = datasets_lvar_data[nrec]
+            rec2 = datasets_hvar_data[nrec]
+            
+#        print ("\n",nrec, permut_l[nrec], permut_h[nrec])
+#        print (rec1['corr2d'].shape,rec2['corr2d'].shape)
+        rec = {'corr2d':           np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
+               'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
+               'gt_ds':            np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
+
+        rec['corr2d'][0::2] =           rec1['corr2d'] 
+        rec['corr2d'][1::2] =           rec2['corr2d'] 
+        
+        rec['target_disparity'][0::2] = rec1['target_disparity'] 
+        rec['target_disparity'][1::2] = rec2['target_disparity'] 
+
+        rec['gt_ds'][0::2] =            rec1['gt_ds'] 
+        rec['gt_ds'][1::2] =            rec2['gt_ds']
+        if del_src:
+            datasets_lvar_data[nrec] = None
+            datasets_hvar_data[nrec] = None
+        datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =       -0.1, #minimal GT disparity
+                 max_disp =       20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =         2.0,
+                 radius =          0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool) 
+    not_nan_ext[radius:-radius,radius:-radius] = not_nan
+    for dy in range(2*radius+1):
+        for dx in range(2*radius+1):
+            not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
+    not_nan = not_nan_ext[radius:-radius,radius:-radius]         
+        
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+data_dir =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" 
+img_dir =   "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+
+dir_train_lvar = data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar = data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_lvar =  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_hvar =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" #  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_img =        os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
+dir_result =     os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
+
+files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+"""
+Try again - all hvar training, train than different hvar testing.
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
+Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
+match for either of hvar/lvar, even those that were used for training
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
+
+try same with higher LR - will they eventually converge?
+
+Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
+"""
+#just testing:
+#files_train_lvar = files_train_hvar
+files_test_lvar =  ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
+files_test_hvar =  ["train004_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
+#files_img =        ['1527257933_150165-v04'] # overlook
+#files_img =        ['1527256858_150165-v01'] # state Street
+files_img =        ['1527256816_150165-v02'] # staete Street
+for i, path in enumerate(files_train_lvar):
+    files_train_lvar[i]=os.path.join(dir_train_lvar, path)
+    
+for i, path in enumerate(files_train_hvar):
+    files_train_hvar[i]=os.path.join(dir_train_hvar, path)
+    
+for i, path in enumerate(files_test_lvar):
+    files_test_lvar[i]=os.path.join(dir_test_lvar, path)
+    
+for i, path in enumerate(files_test_hvar):
+    files_test_hvar[i]=os.path.join(dir_test_hvar, path)
+
+result_files=[]
+for i, path in enumerate(files_img):
+    files_img[i] =      os.path.join(dir_img,    path+'.tfrecords')
+    result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
+
+
+#file_test_hvar=  None
+weight_hvar = 0.26
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+for result_file in result_files:
+    try:
+        eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
+    #exit(0)
+    except:
+        pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":           corr2d,
+                                "target_disparity": target_disparity,
+                                "gt_ds":            gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+datasets_train_hvar = []
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+    """
+if (file_test_lvar):
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_lvar)
+    dataset_test_lvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+if (file_test_hvar):
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(file_test_hvar)
+    dataset_test_hvar = {"corr2d":corr2d,
+                           "target_disparity":target_disparity,
+                           "gt_ds":gt_ds}
+    print_time("  Done")
+    """
+datasets_test_lvar = []
+for fpath in files_test_lvar:
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+datasets_test_hvar = []
+for fpath in files_test_hvar:
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+    
+    
+    
+    
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_lvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_hvar,  TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+print_time("Reshaping test data (low variance)", end="")
+reformat_to_clusters(datasets_test_lvar)
+print_time("  Done")
+print_time("Reshaping test data (high variance)", end="")
+reformat_to_clusters(datasets_test_hvar)
+print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+
+if ZIP_LHVAR:
+    print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
+#    datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
+    datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar) # no shuffle, delete src
+    print_time("  Done")
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+for dataset_test_lvar in datasets_test_lvar:
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+for dataset_test_hvar in datasets_test_hvar:
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+#dataset_tt    TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>    
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
+"""
+next_element_tt    dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}    
+    'corr2d' (140405473715624)    Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)    
+    'gt_ds' (140405473715680)    Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)    
+    'target_disparity' (140405501995888)    Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)    
+"""
+
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i),     reuse = reuse | (j > 0)))
+                   if num_non_sum > 0:
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))    
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           
+    return fc[-1]
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?,9,325]-> [?,25,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+                inter_list.append(network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8))
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2)   
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+    
+corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
+                          tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
+
+out =       network_siam(input=corr2d_Nx325,
+                         layout1 =   NN_LAYOUT1, 
+                         layout2 =   NN_LAYOUT2,
+                         sym8 =      SYM8_SUB,
+                         only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0, # 0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+              
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
+    tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",   tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",   tf_img_test0)
+    tf.summary.scalar("img_test9",   tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs6_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    loss_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_avg = 0.0     
+    train2_avg = 0.0
+    test_avg = 0.0     
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+#    num_train_variants = len(files_train_lvar)
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        print ("sr1",file=sys.stderr,end=" ")
+        if (file_index == 0) and SHUFFLE_FILES:
+            print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
+
+            # datasets_train = zip_lvar_hvar(datasets_train_lvar, datasets_train_hvar, del_src= not SHUFFLE_FILES, shuffle = SHUFFLE_FILES)
+            shuffle_in_place (datasets_train, 0, 2)
+            shuffle_in_place (datasets_train, 1, 2)
+            print_time("  Done")
+            
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        for i in range(dataset_train_size):
+            try:
+                train_summary,_, G_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        G_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+                    ],
+                    feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                loss_train_hist[i] =  G_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+
+        train_avg =       np.average(loss_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        tst_avg =        [0.0]*len(datasets_test)
+        tst2_avg =       [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            for i in range(dataset_test_size):
+                try:
+                    test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         G_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                            feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    loss_test_hist[i] =  G_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+            test_avg =  np.average(loss_test_hist).astype(np.float32)
+            tst_avg[ntest] =  test_avg   
+            test2_avg = np.average(loss2_test_hist).astype(np.float32)
+            tst2_avg[ntest] =  test2_avg   
+            gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                                feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                result_file = result_files[ntest]
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)                
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)

nn_ds_neibs7.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+import sys
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            3e-4 #LR    # 1e-4
+LR200 =            1e-4 #LR100 # 3e-5
+LR400 =            3e-5 #LR200 # 1e-5
+LR600 =            1e-5 #LR400 # 3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     3000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+#EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+TWO_TRAINS =       True # use 2 train sets         
+BATCH_SIZE =       ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =       4 # #4 # 8 # 4 # 0 #3 # 0 #  0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =       4 # 0 # 0 # 4 # 0 # 0 # 3 #  0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =         True # False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+#DEBUG_PACK_TILES = True
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     2 # 1 # 1 - 3x3, 2 - 5x5 tiles
+SHUFFLE_FILES  =     True
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'],           copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'],            copy = False)
+
+def permute_to_swaps(perm):
+    pairs = []
+    for i in range(len(perm)):
+        w = np.where(perm == i)[0][0]
+        if w != i:
+            pairs.append([i,w])
+            perm[w] = perm[i]
+            perm[i] = i
+    return pairs        
+        
+def shuffle_in_place(datasets_data, indx, period):
+    swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
+    num_entries = datasets_data[0]['corr2d'].shape[0] // period
+    for swp in swaps:
+        ds0 = datasets_data[swp[0]]
+        ds1 = datasets_data[swp[1]]
+        tmp = ds0['corr2d'][indx::period].copy() 
+        ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
+        ds1['corr2d'][indx::period] = tmp
+
+        tmp = ds0['target_disparity'][indx::period].copy() 
+        ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
+        ds1['target_disparity'][indx::period] = tmp
+
+        tmp = ds0['gt_ds'][indx::period].copy() 
+        ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
+        ds1['gt_ds'][indx::period] = tmp
+    
+def shuffle_chunks_in_place(datasets_data, tiles_groups_per_chunk):
+    """
+    Improve shuffling by preserving indices inside batches (0 <->0, ... 39 <->39 for 40 tile group batches)
+    """
+    num_files = len(datasets_data)
+    #chunks_per_file = datasets_data[0]['target_disparity']
+    for nf, ds in enumerate(datasets_data):
+        groups_per_file = ds['corr2d'].shape[0]
+        chunks_per_file = groups_per_file//tiles_groups_per_chunk
+        permut = np.random.permutation(chunks_per_file)
+        ds['corr2d'] =           ds['corr2d'].          reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        ds['target_disparity'] = ds['target_disparity'].reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        ds['gt_ds'] =            ds['gt_ds'].           reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        
+    
+    
+def zip_lvar_hvar_old(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+
+    datasets_data = []
+    permut_l = np.random.permutation(len(datasets_lvar_data))    
+    permut_h = np.random.permutation(len(datasets_hvar_data))
+    """
+    TODO: add shuffling batches in each record
+    """    
+    for nrec in range(len(datasets_lvar_data)):
+        if shuffle:
+            rec1 = datasets_lvar_data[permut_l[nrec]]
+            rec2 = datasets_hvar_data[permut_h[nrec]]
+        else:
+            rec1 = datasets_lvar_data[nrec]
+            rec2 = datasets_hvar_data[nrec]
+            
+#        print ("\n",nrec, permut_l[nrec], permut_h[nrec])
+#        print (rec1['corr2d'].shape,rec2['corr2d'].shape)
+        rec = {'corr2d':           np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
+               'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
+               'gt_ds':            np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
+
+        rec['corr2d'][0::2] =           rec1['corr2d'] 
+        rec['corr2d'][1::2] =           rec2['corr2d'] 
+        
+        rec['target_disparity'][0::2] = rec1['target_disparity'] 
+        rec['target_disparity'][1::2] = rec2['target_disparity'] 
+
+        rec['gt_ds'][0::2] =            rec1['gt_ds'] 
+        rec['gt_ds'][1::2] =            rec2['gt_ds']
+        if del_src:
+            datasets_lvar_data[nrec] = None
+            datasets_hvar_data[nrec] = None
+        datasets_data.append(rec)
+    return datasets_data    
+
+def zip_lvar_hvar(datasets_all_data, del_src = True):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    num_sets_to_combine = len(datasets_all_data)
+    datasets_data = []
+    if num_sets_to_combine:
+        for nrec in range(len(datasets_all_data[0])):
+            recs = [[] for _ in range(num_sets_to_combine)]
+            for nset, datasets in enumerate(datasets_all_data):
+                recs[nset] = datasets[nrec]
+                
+            rec = {'corr2d':           np.empty((recs[0]['corr2d'].shape[0]*num_sets_to_combine,          recs[0]['corr2d'].shape[1]),dtype=np.float32),
+                   'target_disparity': np.empty((recs[0]['target_disparity'].shape[0]*num_sets_to_combine,recs[0]['target_disparity'].shape[1]),dtype=np.float32),
+                   'gt_ds':            np.empty((recs[0]['gt_ds'].shape[0]*num_sets_to_combine,           recs[0]['gt_ds'].shape[1]),dtype=np.float32)}
+            
+            for nset, reci in enumerate(recs):
+                rec['corr2d']          [nset::num_sets_to_combine] =  recs[nset]['corr2d'] 
+                rec['target_disparity'][nset::num_sets_to_combine] =  recs[nset]['target_disparity'] 
+                rec['gt_ds']           [nset::num_sets_to_combine] =  recs[nset]['gt_ds'] 
+            if del_src:
+                for nset in range(num_sets_to_combine):
+                    datasets_all_data[nset][nrec] = None
+            datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =       -0.1, #minimal GT disparity
+                 max_disp =       20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =         2.0,
+                 radius =          0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool) 
+    not_nan_ext[radius:-radius,radius:-radius] = not_nan
+    for dy in range(2*radius+1):
+        for dx in range(2*radius+1):
+            not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
+    not_nan = not_nan_ext[radius:-radius,radius:-radius]         
+        
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+data_dir =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" 
+data_dir1 = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+ 
+img_dir =   "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+
+dir_train_lvar =  data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar =  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_lvar1 = data_dir1 #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar1 = data_dir1 # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_lvar =  data_dir # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_hvar =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_3" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" #  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_img =        os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
+dir_result =     os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
+
+files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+files_train_lvar1 = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar1 = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+
+
+
+"""
+Try again - all hvar training, train than different hvar testing.
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
+Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
+match for either of hvar/lvar, even those that were used for training
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
+
+try same with higher LR - will they eventually converge?
+
+Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
+"""
+#just testing:
+#files_train_lvar = files_train_hvar
+files_test_lvar =  ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
+files_test_hvar =  ["train000_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
+#files_img =        ['1527257933_150165-v04'] # overlook
+#files_img =        ['1527256858_150165-v01'] # state Street
+files_img =        ['1527256816_150165-v02'] # staete Street
+for i, path in enumerate(files_train_lvar):
+    files_train_lvar[i]=os.path.join(dir_train_lvar, path)
+    
+for i, path in enumerate(files_train_hvar):
+    files_train_hvar[i]=os.path.join(dir_train_hvar, path)
+
+# Second set of files
+
+for i, path in enumerate(files_train_lvar1):
+    files_train_lvar1[i]=os.path.join(dir_train_lvar1, path)
+    
+for i, path in enumerate(files_train_hvar1):
+    files_train_hvar1[i]=os.path.join(dir_train_hvar1, path)
+
+
+    
+for i, path in enumerate(files_test_lvar):
+    files_test_lvar[i]=os.path.join(dir_test_lvar, path)
+    
+for i, path in enumerate(files_test_hvar):
+    files_test_hvar[i]=os.path.join(dir_test_hvar, path)
+
+result_files=[]
+for i, path in enumerate(files_img):
+    files_img[i] =      os.path.join(dir_img,    path+'.tfrecords')
+    result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
+
+
+#file_test_hvar=  None
+weight_hvar = 0.26
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+for result_file in result_files:
+    try:
+        eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
+    #exit(0)
+    except:
+        pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":           corr2d,
+                                "target_disparity": target_disparity,
+                                "gt_ds":            gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar =  []
+datasets_train_hvar =  []
+datasets_train_lvar1 = []
+datasets_train_hvar1 = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+if TWO_TRAINS:
+    for fpath in files_train_lvar1:
+        print_time("Importing train data1 (low variance) from "+fpath, end="")
+        corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+        datasets_train_lvar1.append({"corr2d":corr2d,
+                                     "target_disparity":target_disparity,
+                                     "gt_ds":gt_ds})
+        print_time("  Done")
+    for fpath in files_train_hvar1:
+        print_time("Importing train data1 (high variance) from "+fpath, end="")
+        corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+        datasets_train_hvar1.append({"corr2d":corr2d,
+                                     "target_disparity":target_disparity,
+                                     "gt_ds":gt_ds})
+        print_time("  Done")
+
+
+
+datasets_test_lvar = []
+for fpath in files_test_lvar:
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+datasets_test_hvar = []
+for fpath in files_test_hvar:
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+    
+    
+    
+    
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_lvar1, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar1, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_lvar,   TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_hvar,   TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+
+print_time("Reshaping train data1 (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar1)
+print_time("  Done")
+print_time("Reshaping train data1 (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar1)
+print_time("  Done")
+
+print_time("Reshaping test data (low variance)", end="")
+reformat_to_clusters(datasets_test_lvar)
+print_time("  Done")
+print_time("Reshaping test data (high variance)", end="")
+reformat_to_clusters(datasets_test_hvar)
+print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+datasets_train_list =      [datasets_train_lvar,  datasets_train_hvar]
+if TWO_TRAINS:
+    datasets_train_list += [datasets_train_lvar1, datasets_train_hvar1]
+
+if ZIP_LHVAR:
+    print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
+    datasets_train = zip_lvar_hvar(datasets_train_list, del_src = True) # no shuffle, delete src
+    print_time("  Done")
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+for dataset_test_lvar in datasets_test_lvar:
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+for dataset_test_hvar in datasets_test_hvar:
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+#dataset_tt    TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>    
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
+"""
+next_element_tt    dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}    
+    'corr2d' (140405473715624)    Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)    
+    'gt_ds' (140405473715680)    Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)    
+    'target_disparity' (140405501995888)    Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)    
+"""
+
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope='g_fc_sub'+str(i),     reuse = reuse | (j > 0)))
+                   if num_non_sum > 0:
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope='g_fc_sub'+str(i)+"r", reuse = reuse))    
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           
+    return fc[-1]
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?,9,325]-> [?,25,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+                inter_list.append(network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8))
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2)   
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+    
+corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
+                          tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
+
+out =       network_siam(input=corr2d_Nx325,
+                         layout1 =   NN_LAYOUT1, 
+                         layout2 =   NN_LAYOUT2,
+                         sym8 =      SYM8_SUB,
+                         only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0, # 0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+              
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss = tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 = tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 = tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("G_loss_epoch", tf_ph_G_loss)
+    tf.summary.scalar("sq_diff_epoch",tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",   tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",   tf_img_test0)
+    tf.summary.scalar("img_test9",   tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs7_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    loss_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_avg = 0.0     
+    train2_avg = 0.0
+    test_avg = 0.0     
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        print ("sr1",file=sys.stderr,end=" ")
+        if (file_index == 0) and SHUFFLE_FILES:
+            num_sets = len(datasets_train_list)
+            print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
+            for i in range(num_sets):
+                shuffle_in_place (datasets_train, i, num_sets)
+            print_time("  Done")
+            print_time("Shuffling tile chunks ", end="")
+            shuffle_chunks_in_place (datasets_train, 1)
+            print_time("  Done")
+            
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        for i in range(dataset_train_size):
+            try:
+                train_summary,_, G_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        G_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+                    ],
+                    feed_dict={lr:learning_rate,tf_ph_G_loss:train_avg, tf_ph_sq_diff:train2_avg, tf_gtvar_diff:gtvar_train_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                loss_train_hist[i] =  G_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+
+        train_avg =       np.average(loss_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        tst_avg =        [0.0]*len(datasets_test)
+        tst2_avg =       [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            for i in range(dataset_test_size):
+                try:
+                    test_summaries[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         G_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                            feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    loss_test_hist[i] =  G_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+            test_avg =  np.average(loss_test_hist).astype(np.float32)
+            tst_avg[ntest] =  test_avg   
+            test2_avg = np.average(loss2_test_hist).astype(np.float32)
+            tst2_avg[ntest] =  test2_avg   
+            gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                                feed_dict={lr:learning_rate,tf_ph_G_loss:test_avg, tf_ph_sq_diff:test2_avg, tf_gtvar_diff:gtvar_test_avg, tf_img_test0:img_gain_test0, tf_img_test9:img_gain_test9})  # previous value of *_avg
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                result_file = result_files[ntest]
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)                
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)

nn_ds_neibs8.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+import sys
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            3e-4 #LR    # 1e-4
+LR200 =            1e-4 #LR100 # 3e-5
+LR400 =            3e-5 #LR200 # 1e-5
+LR600 =            1e-5 #LR400 # 3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     3000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+#EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+TWO_TRAINS =       True # use 2 train sets         
+BATCH_SIZE =       ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =       0 # 4 # #4 # 8 # 4 # 0 #3 # 0 #  0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =       0 # 4 # 0 # 0 # 4 # 0 # 0 # 3 #  0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =        False #  True # False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+#DEBUG_PACK_TILES = True
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     2 # 1 # 1 - 3x3, 2 - 5x5 tiles
+SHUFFLE_FILES  =     True
+WLOSS_LAMBDA =       50.0 # 5.0 # 1.0 # fraction of the W_loss (input layers weight non-uniformity) added to G_loss
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])+"LMBD"+str(WLOSS_LAMBDA)
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'],           copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'],            copy = False)
+
+def permute_to_swaps(perm):
+    pairs = []
+    for i in range(len(perm)):
+        w = np.where(perm == i)[0][0]
+        if w != i:
+            pairs.append([i,w])
+            perm[w] = perm[i]
+            perm[i] = i
+    return pairs        
+        
+def shuffle_in_place(datasets_data, indx, period):
+    swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
+    num_entries = datasets_data[0]['corr2d'].shape[0] // period
+    for swp in swaps:
+        ds0 = datasets_data[swp[0]]
+        ds1 = datasets_data[swp[1]]
+        tmp = ds0['corr2d'][indx::period].copy() 
+        ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
+        ds1['corr2d'][indx::period] = tmp
+
+        tmp = ds0['target_disparity'][indx::period].copy() 
+        ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
+        ds1['target_disparity'][indx::period] = tmp
+
+        tmp = ds0['gt_ds'][indx::period].copy() 
+        ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
+        ds1['gt_ds'][indx::period] = tmp
+    
+def shuffle_chunks_in_place(datasets_data, tiles_groups_per_chunk):
+    """
+    Improve shuffling by preserving indices inside batches (0 <->0, ... 39 <->39 for 40 tile group batches)
+    """
+    num_files = len(datasets_data)
+    #chunks_per_file = datasets_data[0]['target_disparity']
+    for nf, ds in enumerate(datasets_data):
+        groups_per_file = ds['corr2d'].shape[0]
+        chunks_per_file = groups_per_file//tiles_groups_per_chunk
+        permut = np.random.permutation(chunks_per_file)
+        ds['corr2d'] =           ds['corr2d'].          reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        ds['target_disparity'] = ds['target_disparity'].reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        ds['gt_ds'] =            ds['gt_ds'].           reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        
+    
+    
+def zip_lvar_hvar_old(datasets_lvar_data, datasets_hvar_data, del_src = True, shuffle = False):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+
+    datasets_data = []
+    permut_l = np.random.permutation(len(datasets_lvar_data))    
+    permut_h = np.random.permutation(len(datasets_hvar_data))
+    """
+    TODO: add shuffling batches in each record
+    """    
+    for nrec in range(len(datasets_lvar_data)):
+        if shuffle:
+            rec1 = datasets_lvar_data[permut_l[nrec]]
+            rec2 = datasets_hvar_data[permut_h[nrec]]
+        else:
+            rec1 = datasets_lvar_data[nrec]
+            rec2 = datasets_hvar_data[nrec]
+            
+#        print ("\n",nrec, permut_l[nrec], permut_h[nrec])
+#        print (rec1['corr2d'].shape,rec2['corr2d'].shape)
+        rec = {'corr2d':           np.empty((rec1['corr2d'].shape[0]*2,rec1['corr2d'].shape[1]),dtype=np.float32),
+               'target_disparity': np.empty((rec1['target_disparity'].shape[0]*2,rec1['target_disparity'].shape[1]),dtype=np.float32),
+               'gt_ds':            np.empty((rec1['gt_ds'].shape[0]*2,rec1['gt_ds'].shape[1]),dtype=np.float32)}
+
+        rec['corr2d'][0::2] =           rec1['corr2d'] 
+        rec['corr2d'][1::2] =           rec2['corr2d'] 
+        
+        rec['target_disparity'][0::2] = rec1['target_disparity'] 
+        rec['target_disparity'][1::2] = rec2['target_disparity'] 
+
+        rec['gt_ds'][0::2] =            rec1['gt_ds'] 
+        rec['gt_ds'][1::2] =            rec2['gt_ds']
+        if del_src:
+            datasets_lvar_data[nrec] = None
+            datasets_hvar_data[nrec] = None
+        datasets_data.append(rec)
+    return datasets_data    
+
+def zip_lvar_hvar(datasets_all_data, del_src = True):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    num_sets_to_combine = len(datasets_all_data)
+    datasets_data = []
+    if num_sets_to_combine:
+        for nrec in range(len(datasets_all_data[0])):
+            recs = [[] for _ in range(num_sets_to_combine)]
+            for nset, datasets in enumerate(datasets_all_data):
+                recs[nset] = datasets[nrec]
+                
+            rec = {'corr2d':           np.empty((recs[0]['corr2d'].shape[0]*num_sets_to_combine,          recs[0]['corr2d'].shape[1]),dtype=np.float32),
+                   'target_disparity': np.empty((recs[0]['target_disparity'].shape[0]*num_sets_to_combine,recs[0]['target_disparity'].shape[1]),dtype=np.float32),
+                   'gt_ds':            np.empty((recs[0]['gt_ds'].shape[0]*num_sets_to_combine,           recs[0]['gt_ds'].shape[1]),dtype=np.float32)}
+            
+            for nset, reci in enumerate(recs):
+                rec['corr2d']          [nset::num_sets_to_combine] =  recs[nset]['corr2d'] 
+                rec['target_disparity'][nset::num_sets_to_combine] =  recs[nset]['target_disparity'] 
+                rec['gt_ds']           [nset::num_sets_to_combine] =  recs[nset]['gt_ds'] 
+            if del_src:
+                for nset in range(num_sets_to_combine):
+                    datasets_all_data[nset][nrec] = None
+            datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =       -0.1, #minimal GT disparity
+                 max_disp =       20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =         2.0,
+                 radius =          0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool) 
+    not_nan_ext[radius:-radius,radius:-radius] = not_nan
+    for dy in range(2*radius+1):
+        for dx in range(2*radius+1):
+            not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
+    not_nan = not_nan_ext[radius:-radius,radius:-radius]         
+        
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+data_dir =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" 
+data_dir1 = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+ 
+img_dir =   "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+
+dir_train_lvar =  data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar =  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_lvar1 = data_dir1 #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar1 = data_dir1 # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_lvar =  data_dir # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_hvar =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_3" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" #  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_img =        os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
+dir_result =     os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
+
+files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+files_train_lvar1 = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar1 = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+
+
+
+"""
+Try again - all hvar training, train than different hvar testing.
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
+Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
+match for either of hvar/lvar, even those that were used for training
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
+
+try same with higher LR - will they eventually converge?
+
+Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
+"""
+#just testing:
+#files_train_lvar = files_train_hvar
+files_test_lvar =  ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
+files_test_hvar =  ["train000_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
+#files_img =        ['1527257933_150165-v04'] # overlook
+#files_img =        ['1527256858_150165-v01'] # state Street
+files_img =        ['1527256816_150165-v02'] # staete Street
+for i, path in enumerate(files_train_lvar):
+    files_train_lvar[i]=os.path.join(dir_train_lvar, path)
+    
+for i, path in enumerate(files_train_hvar):
+    files_train_hvar[i]=os.path.join(dir_train_hvar, path)
+
+# Second set of files
+
+for i, path in enumerate(files_train_lvar1):
+    files_train_lvar1[i]=os.path.join(dir_train_lvar1, path)
+    
+for i, path in enumerate(files_train_hvar1):
+    files_train_hvar1[i]=os.path.join(dir_train_hvar1, path)
+
+
+    
+for i, path in enumerate(files_test_lvar):
+    files_test_lvar[i]=os.path.join(dir_test_lvar, path)
+    
+for i, path in enumerate(files_test_hvar):
+    files_test_hvar[i]=os.path.join(dir_test_hvar, path)
+
+result_files=[]
+for i, path in enumerate(files_img):
+    files_img[i] =      os.path.join(dir_img,    path+'.tfrecords')
+    result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
+
+
+#file_test_hvar=  None
+weight_hvar = 0.26
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+for result_file in result_files:
+    try:
+        eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
+    #exit(0)
+    except:
+        pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":           corr2d,
+                                "target_disparity": target_disparity,
+                                "gt_ds":            gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar =  []
+datasets_train_hvar =  []
+datasets_train_lvar1 = []
+datasets_train_hvar1 = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+if TWO_TRAINS:
+    for fpath in files_train_lvar1:
+        print_time("Importing train data1 (low variance) from "+fpath, end="")
+        corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+        datasets_train_lvar1.append({"corr2d":corr2d,
+                                     "target_disparity":target_disparity,
+                                     "gt_ds":gt_ds})
+        print_time("  Done")
+    for fpath in files_train_hvar1:
+        print_time("Importing train data1 (high variance) from "+fpath, end="")
+        corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+        datasets_train_hvar1.append({"corr2d":corr2d,
+                                     "target_disparity":target_disparity,
+                                     "gt_ds":gt_ds})
+        print_time("  Done")
+
+
+
+datasets_test_lvar = []
+for fpath in files_test_lvar:
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+datasets_test_hvar = []
+for fpath in files_test_hvar:
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+    
+    
+    
+    
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_lvar1, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar1, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_lvar,   TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_hvar,   TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+
+print_time("Reshaping train data1 (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar1)
+print_time("  Done")
+print_time("Reshaping train data1 (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar1)
+print_time("  Done")
+
+print_time("Reshaping test data (low variance)", end="")
+reformat_to_clusters(datasets_test_lvar)
+print_time("  Done")
+print_time("Reshaping test data (high variance)", end="")
+reformat_to_clusters(datasets_test_hvar)
+print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+datasets_train_list =      [datasets_train_lvar,  datasets_train_hvar]
+if TWO_TRAINS:
+    datasets_train_list += [datasets_train_lvar1, datasets_train_hvar1]
+
+if ZIP_LHVAR:
+    print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
+    datasets_train = zip_lvar_hvar(datasets_train_list, del_src = True) # no shuffle, delete src
+    print_time("  Done")
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+for dataset_test_lvar in datasets_test_lvar:
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+for dataset_test_hvar in datasets_test_hvar:
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+#dataset_tt    TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>    
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
+"""
+next_element_tt    dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}    
+    'corr2d' (140405473715624)    Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)    
+    'gt_ds' (140405473715680)    Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)    
+    'target_disparity' (140405501995888)    Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)    
+"""
+
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    inp_weights = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       reuse_this = reuse | (j > 0)
+                       scp = 'g_fc_sub'+str(i)
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope= scp,     reuse = reuse_this))
+                       if not reuse_this:
+                           with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
+                              inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs])) 
+                   if num_non_sum > 0:
+                       reuse_this = reuse
+                       scp = 'g_fc_sub'+str(i)+"r"
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope=scp, reuse = reuse_this))    
+                       if not reuse_this:
+                           with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
+                              inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs])) 
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   scp = 'g_fc_sub'+str(i)
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope= scp, reuse = reuse))
+                   if not reuse:
+                       with tf.variable_scope(scp, reuse=True) : # tf.AUTO_REUSE):
+                          inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs])) 
+           
+    return fc[-1], inp_weights
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?,9,325]-> [?,25,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        inp_weights = []
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+#                inter_list.append(network_sub(input[:,i,:],
+#                                          layout= layout1,
+#                                          reuse= reuse,
+#                                          sym8 = sym8))
+                ns, ns_weights = network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8)
+                inter_list.append(ns)
+                inp_weights += ns_weights
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2),  inp_weights 
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        
+        
+def weightsLoss(inp_weights):       # [batch_size,(1..2)] tf_result
+#                weights_lambdas):  # single lambda or same length as inp_weights.shape[1]
+    """
+    Enforcing 'smooth' weights for the input 2d correlation tiles
+    @return mean squared difference for each weight and average of 8 neighbors divided by mean squared weights
+    """
+    weight_ortho = 1.0
+    weight_diag  = 0.7
+    sw = 4.0 * (weight_ortho + weight_diag)
+    weight_ortho /= sw
+    weight_diag /=  sw
+#    w_neib = tf.const([[weight_diag,  weight_ortho, weight_diag],
+#                       [weight_ortho, -1.0,         weight_ortho],
+#                       [weight_diag,  weight_ortho, weight_diag]])
+    with tf.name_scope("WeightsLoss"):
+        # Adding 1 tile border
+        tf_inp =     tf.reshape(inp_weights[:TILE_LAYERS * TILE_SIZE,:], [TILE_LAYERS, FILE_TILE_SIDE, FILE_TILE_SIDE, inp_weights.shape[1]], name = "tf_inp")
+        tf_inp_ext_h = tf.concat([tf_inp       [:, :,  :1, :], tf_inp,       tf_inp      [:,   :, -1:, :]], axis = 2, name ="tf_inp_ext_h")
+        tf_inp_ext   = tf.concat([tf_inp_ext_h [:, :1, :,  :], tf_inp_ext_h, tf_inp_ext_h[:, -1:,   :, :]], axis = 1, name ="tf_inp_ext")
+        s_ortho = tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:,:] + tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:, :] 
+        s_corn =  tf_inp_ext[:, :-2,:-2,:] + tf_inp_ext[:, :-2, 2:,:] + tf_inp_ext[:,2:,  :-2,:] + tf_inp_ext[:,2:  , 2:, :]
+        w_diff =  tf.subtract(tf_inp, s_ortho * weight_ortho + s_corn * weight_diag, name="w_diff") 
+        w_diff2 = tf.multiply(w_diff, w_diff,                                        name="w_diff2") 
+        w_var =   tf.reduce_mean(w_diff2,                                            name="w_var")
+        w2_mean = tf.reduce_mean(inp_weights * inp_weights,                          name="w2_mean")
+        w_rel =   tf.divide(w_var, w2_mean,                                          name= "w_rel")
+        return w_rel # scalar, cost for weights non-smoothness in 2d
+        
+        
+    
+corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
+                          tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
+
+out, inp_weights =       network_siam(input=corr2d_Nx325,
+                                      layout1 =   NN_LAYOUT1, 
+                                      layout2 =   NN_LAYOUT2,
+                                      sym8 =      SYM8_SUB,
+                                      only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0, # 0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+
+if WLOSS_LAMBDA > 0.0:   
+    W_loss =     weightsLoss(inp_weights[0]) #    inp_weights - list of tensors, currently - just [0]
+    GW_loss =    tf.add(G_loss, WLOSS_LAMBDA * W_loss, name = "GW_loss")
+else:
+    GW_loss = G_loss
+    W_loss =     0.0
+          
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss =  tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_W_loss =  tf.placeholder(tf.float32,shape=None,name='W_loss_avg')
+tf_ph_GW_loss = tf.placeholder(tf.float32,shape=None,name='GW_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 =  tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 =  tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("GW_loss",      GW_loss)
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("W_loss",       W_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("GW_loss_epoch", tf_ph_GW_loss)
+    tf.summary.scalar("G_loss_epoch",  tf_ph_G_loss)
+    tf.summary.scalar("W_loss_epoch",  tf_ph_W_loss)
+    tf.summary.scalar("sq_diff_epoch", tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",    tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",     tf_img_test0)
+    tf.summary.scalar("img_test9",     tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(GW_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs8_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    
+    loss_gw_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_g_train_hist=   np.empty(dataset_train_size, dtype=np.float32)
+    loss_w_train_hist=   np.empty(dataset_train_size, dtype=np.float32)
+    
+    loss_gw_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    loss_g_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss_w_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_gw_avg = 0.0
+    train_g_avg =  0.0
+    train_w_avg =  0.0
+    test_gw_avg =  0.0     
+    test_g_avg =   0.0     
+    test_w_avg =   0.0     
+
+    train2_avg = 0.0
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        print ("sr1",file=sys.stderr,end=" ")
+        if (file_index == 0) and SHUFFLE_FILES:
+            num_sets = len(datasets_train_list)
+            print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
+            for i in range(num_sets):
+                shuffle_in_place (datasets_train, i, num_sets)
+            print_time("  Done")
+            print_time("Shuffling tile chunks ", end="")
+            shuffle_chunks_in_place (datasets_train, 1)
+            print_time("  Done")
+            
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        for i in range(dataset_train_size):
+            try:
+                train_summary,_, GW_loss_trained,  G_loss_trained,  W_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        GW_loss,
+                        G_loss,
+                        W_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+                    ],
+                    feed_dict={lr:            learning_rate,
+                               tf_ph_GW_loss: train_gw_avg,
+                               tf_ph_G_loss:  train_g_avg,
+                               tf_ph_W_loss:  train_w_avg,
+                               tf_ph_sq_diff: train2_avg,
+                               tf_gtvar_diff: gtvar_train_avg,
+                               tf_img_test0:  img_gain_test0,
+                               tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                loss_gw_train_hist[i] = GW_loss_trained
+                loss_g_train_hist[i] =  G_loss_trained
+                loss_w_train_hist[i] =  W_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+
+        train_gw_avg =      np.average(loss_gw_train_hist).astype(np.float32)     
+        train_g_avg =       np.average(loss_g_train_hist).astype(np.float32)     
+        train_w_avg =       np.average(loss_w_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        tst_avg =        [0.0]*len(datasets_test)
+        tst2_avg =       [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            for i in range(dataset_test_size):
+                try:
+                    test_summaries[ntest], GW_loss_tested, G_loss_tested, W_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         GW_loss,
+                         G_loss,
+                         W_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                         feed_dict={lr:            learning_rate,
+                                    tf_ph_GW_loss:  test_gw_avg,
+                                    tf_ph_G_loss:   test_g_avg,
+                                    tf_ph_W_loss:   test_w_avg,
+                                    tf_ph_sq_diff:  test2_avg,
+                                    tf_gtvar_diff:  gtvar_test_avg,
+                                    tf_img_test0:   img_gain_test0,
+                                    tf_img_test9:   img_gain_test9})  # previous value of *_avg
+                    loss_gw_test_hist[i] =  GW_loss_tested
+                    loss_g_test_hist[i] =   G_loss_tested
+                    loss_w_test_hist[i] =   W_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+            test_gw_avg =  np.average(loss_gw_test_hist).astype(np.float32)
+            test_g_avg =  np.average(loss_g_test_hist).astype(np.float32)
+            test_w_avg =  np.average(loss_w_test_hist).astype(np.float32)
+            tst_avg[ntest] =  test_gw_avg   
+            test2_avg = np.average(loss2_test_hist).astype(np.float32)
+            tst2_avg[ntest] =  test2_avg   
+            gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_gw_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                             feed_dict={lr:            learning_rate,
+                                        tf_ph_GW_loss:  test_gw_avg,
+                                        tf_ph_G_loss:   test_g_avg,
+                                        tf_ph_W_loss:   test_w_avg,
+                                        tf_ph_sq_diff: test2_avg,
+                                        tf_gtvar_diff: gtvar_test_avg,
+                                        tf_img_test0:  img_gain_test0,
+                                        tf_img_test9:  img_gain_test9})  # previous value of *_avg
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                result_file = result_files[ntest]
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)                
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)

nn_ds_neibs9.py

+#!/usr/bin/env python3
+from numpy import float64
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import npairs_loss
+from debian.deb822 import PdiffIndex
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+
+from PIL import Image
+
+import os
+import sys
+import glob
+
+import numpy as np
+import itertools
+
+import time
+
+import matplotlib.pyplot as plt
+
+import shutil
+import sys
+
+TIME_START = time.time()
+TIME_LAST  = TIME_START
+DEBUG_LEVEL= 1
+DISP_BATCH_BINS =   20 # Number of batch disparity bins
+STR_BATCH_BINS =    10 # Number of batch strength bins
+FILES_PER_SCENE =    5 # number of random offset files for the scene to select from (0 - use all available)
+#MIN_BATCH_CHOICES = 10 # minimal number of tiles in a file for each bin to select from 
+#MAX_BATCH_FILES =   10 #maximal number of files to use in a batch
+#MAX_EPOCH =        500
+LR =               1e-3 # learning rate
+LR100 =            3e-4 #LR    # 1e-4
+LR200 =            1e-4 #LR100 # 3e-5
+LR400 =            3e-5 #LR200 # 1e-5
+LR600 =            1e-5 #LR400 # 3e-6
+USE_CONFIDENCE =     False
+ABSOLUTE_DISPARITY = False # True # False # True # False
+DEBUG_PLT_LOSS =     True
+TILE_LAYERS =        4
+FILE_TILE_SIDE =     9
+TILE_SIDE =          9 # 7
+TILE_SIZE =         TILE_SIDE* TILE_SIDE # == 81
+FEATURES_PER_TILE =  TILE_LAYERS * TILE_SIZE# == 324
+EPOCHS_TO_RUN =     3000#0 #0
+EPOCHS_FULL_TEST =   5 # 10 # 25# repeat full image test after this number of epochs
+#EPOCHS_SAME_FILE =   20
+RUN_TOT_AVG =       100 # last batches to average. Epoch is 307 training  batches  
+#BATCH_SIZE =       2*1080//9 # == 120 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+TWO_TRAINS =       True # use 2 train sets         
+BATCH_SIZE =       ([1,2][TWO_TRAINS])*2*1000//25 # == 80 Each batch of tiles has balanced D/S tiles, shuffled batches but not inside batches
+SHUFFLE_EPOCH =    True
+NET_ARCH1 =       0 # 4 # #4 # 8 # 4 # 0 #3 # 0 #  0 # 0 # 0 # 8 # 0 # 7 # 2 #0 # 6 #0 # 4 # 3 # overwrite with argv?
+NET_ARCH2 =       0 # 4 # 0 # 0 # 4 # 0 # 0 # 3 #  0 # 3 # 0 # 3 # 0 # 2 #0 # 6 # 0 # 3 # overwrite with argv?
+SYM8_SUB =        True #False #  True # False # True # False # True # False # enforce inputs from 2d correlation have symmetrical ones (groups of 8)
+ONLY_TILE =        None # 4 # None # 0 # 4# None # (remove all but center tile data), put None here for normal operation)
+ZIP_LHVAR =        True # combine _lvar and _hvar as odd/even elements 
+#DEBUG_PACK_TILES = True
+# CLUSTER_RADIUS should match input data
+CLUSTER_RADIUS =     2 # 1 # 1 - 3x3, 2 - 5x5 tiles
+SHUFFLE_FILES  =     True
+WLOSS_LAMBDA =       50.0 # 5.0 # 1.0 # fraction of the W_loss (input layers weight non-uniformity) added to G_loss
+SUFFIX=str(NET_ARCH1)+'-'+str(NET_ARCH2)+ (["R","A"][ABSOLUTE_DISPARITY]) +(["NS","S8"][SYM8_SUB])+"LMBD"+str(WLOSS_LAMBDA)
+
+NN_LAYOUTS = {0:[0,   0,   0,   32,  20,  16],
+              1:[0,   0,   0,  256, 128,  64],
+              2:[0, 128,  32,   32,  32,  16],
+              3:[0,   0,  40,   32,  20,  16],
+              4:[0,   0,   0,    0,  16,  16],
+              5:[0,   0,  64,   32,  32,  16],
+              6:[0,   0,  32,   16,  16,  16],
+              7:[0,   0,  64,   16,  16,  16],
+              8:[0,   0,   0,   64,  20,  16],
+              }
+NN_LAYOUT1 = NN_LAYOUTS[NET_ARCH1]
+NN_LAYOUT2 = NN_LAYOUTS[NET_ARCH2]
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time(txt="",end="\n"):
+    global TIME_LAST
+    t = time.time()
+    if txt:
+        txt +=" "
+    print(("%s"+bcolors.BOLDWHITE+"at %.4fs (+%.4fs)"+bcolors.ENDC)%(txt,t-TIME_START,t-TIME_LAST), end = end, flush=True)
+    TIME_LAST = t
+#reading to memory (testing)
+def readTFRewcordsEpoch(train_filename):
+#    filenames = [train_filename]
+#    dataset = tf.data.TFRecorDataset(filenames)
+    if not  '.tfrecords' in train_filename:
+        train_filename += '.tfrecords'
+    npy_dir_name = "npy"
+    dirname = os.path.dirname(train_filename) 
+    npy_dir = os.path.join(dirname, npy_dir_name)
+    filebasename, file_extension = os.path.splitext(train_filename)
+    filebasename = os.path.basename(filebasename)
+    file_corr2d =           os.path.join(npy_dir,filebasename + '_corr2d.npy')
+    file_target_disparity = os.path.join(npy_dir,filebasename + '_target_disparity.npy')
+    file_gt_ds =            os.path.join(npy_dir,filebasename + '_gt_ds.npy')
+    if (os.path.exists(file_corr2d) and
+        os.path.exists(file_target_disparity) and
+        os.path.exists(file_gt_ds)):
+        corr2d=            np.load (file_corr2d)
+        target_disparity = np.load(file_target_disparity)
+        gt_ds =            np.load(file_gt_ds)
+        pass
+    else:     
+        record_iterator = tf.python_io.tf_record_iterator(path=train_filename)
+        corr2d_list=[]
+        target_disparity_list=[]
+        gt_ds_list = []
+        for string_record in record_iterator:
+            example = tf.train.Example()
+            example.ParseFromString(string_record)
+            corr2d_list.append           (np.array(example.features.feature['corr2d'].float_list.value, dtype=np.float32))
+            target_disparity_list.append (np.array(example.features.feature['target_disparity'].float_list.value, dtype=np.float32))
+            gt_ds_list.append            (np.array(example.features.feature['gt_ds'].float_list.value, dtype= np.float32))
+            pass
+        corr2d=            np.array(corr2d_list)
+        target_disparity = np.array(target_disparity_list)
+        gt_ds =            np.array(gt_ds_list)
+        try:
+            os.makedirs(os.path.dirname(file_corr2d))
+        except:
+            pass     
+
+        np.save(file_corr2d,           corr2d)
+        np.save(file_target_disparity, target_disparity)
+        np.save(file_gt_ds,            gt_ds)
+        
+    return corr2d, target_disparity, gt_ds   
+
+#from http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
+def read_and_decode(filename_queue):
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+
+    features = tf.parse_single_example(
+      serialized_example,
+      # Defaults are not specified since both keys are required.
+      features={
+        'corr2d':           tf.FixedLenFeature([FEATURES_PER_TILE],tf.float32), #string),
+        'target_disparity': tf.FixedLenFeature([1],   tf.float32), #.string),
+        'gt_ds':            tf.FixedLenFeature([2],  tf.float32)  #.string)
+        })
+    corr2d =           features['corr2d'] # tf.decode_raw(features['corr2d'], tf.float32)
+    target_disparity = features['target_disparity'] # tf.decode_raw(features['target_disparity'], tf.float32)
+    gt_ds =            tf.cast(features['gt_ds'], tf.float32) # tf.decode_raw(features['gt_ds'], tf.float32)
+    in_features = tf.concat([corr2d,target_disparity],0)
+    # still some nan-s in correlation data?
+#    in_features_clean = tf.where(tf.is_nan(in_features), tf.zeros_like(in_features), in_features)     
+#    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features_clean, target_disparity, gt_ds],
+    corr2d_out, target_disparity_out, gt_ds_out = tf.train.shuffle_batch( [in_features, target_disparity, gt_ds],
+                                                 batch_size=1000, # 2,
+                                                 capacity=30,
+                                                 num_threads=2,
+                                                 min_after_dequeue=10)
+    return corr2d_out, target_disparity_out, gt_ds_out
+#http://adventuresinmachinelearning.com/introduction-tensorflow-queuing/
+def add_margins(npa,radius, val = np.nan):
+    npa_ext = np.empty((npa.shape[0]+2*radius, npa.shape[1]+2*radius, npa.shape[2]), dtype = npa.dtype)
+    npa_ext[radius:radius + npa.shape[0],radius:radius + npa.shape[1]] = npa
+    npa_ext[0:radius,:,:] = val  
+    npa_ext[radius + npa.shape[0]:,:,:] = val  
+    npa_ext[:,0:radius,:] = val  
+    npa_ext[:, radius + npa.shape[1]:,:] = val
+    return npa_ext   
+
+def add_neibs(npa_ext,radius):
+    height = npa_ext.shape[0]-2*radius
+    width =  npa_ext.shape[1]-2*radius
+    side = 2 * radius + 1
+    size = side * side
+    npa_neib = np.empty((height,  width, side, side, npa_ext.shape[2]), dtype = npa_ext.dtype)
+    for dy in range (side):
+        for dx in range (side):
+            npa_neib[:,:,dy, dx,:]= npa_ext[dy:dy+height, dx:dx+width]
+    return npa_neib.reshape(height, width, -1)    
+    
+def extend_img_to_clusters(datasets_img,radius):
+    side = 2 * radius + 1
+    size = side * side
+    width =  324
+    if len(datasets_img) ==0:
+        return
+    num_tiles = datasets_img[0]['corr2d'].shape[0]
+    height = num_tiles // width 
+    for rec in datasets_img:
+        rec['corr2d'] =           add_neibs(add_margins(rec['corr2d'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['target_disparity'] = add_neibs(add_margins(rec['target_disparity'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1)) 
+        rec['gt_ds'] =            add_neibs(add_margins(rec['gt_ds'].reshape((height,width,-1)), radius, np.nan), radius).reshape((num_tiles,-1))
+        pass
+
+
+
+def reformat_to_clusters(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        rec['corr2d'] =           rec['corr2d'].reshape(          (rec['corr2d'].shape[0]//cluster_size,           rec['corr2d'].shape[1] * cluster_size)) 
+        rec['target_disparity'] = rec['target_disparity'].reshape((rec['target_disparity'].shape[0]//cluster_size, rec['target_disparity'].shape[1] * cluster_size)) 
+        rec['gt_ds'] =            rec['gt_ds'].reshape(           (rec['gt_ds'].shape[0]//cluster_size,            rec['gt_ds'].shape[1] * cluster_size))
+
+def replace_nan(datasets_data):
+    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    for rec in datasets_data:
+        np.nan_to_num(rec['corr2d'],           copy = False) 
+        np.nan_to_num(rec['target_disparity'], copy = False) 
+        np.nan_to_num(rec['gt_ds'],            copy = False)
+
+def permute_to_swaps(perm):
+    pairs = []
+    for i in range(len(perm)):
+        w = np.where(perm == i)[0][0]
+        if w != i:
+            pairs.append([i,w])
+            perm[w] = perm[i]
+            perm[i] = i
+    return pairs        
+        
+def shuffle_in_place(datasets_data, indx, period):
+    swaps = permute_to_swaps(np.random.permutation(len(datasets_data)))
+    num_entries = datasets_data[0]['corr2d'].shape[0] // period
+    for swp in swaps:
+        ds0 = datasets_data[swp[0]]
+        ds1 = datasets_data[swp[1]]
+        tmp = ds0['corr2d'][indx::period].copy() 
+        ds0['corr2d'][indx::period] = ds1['corr2d'][indx::period]
+        ds1['corr2d'][indx::period] = tmp
+
+        tmp = ds0['target_disparity'][indx::period].copy() 
+        ds0['target_disparity'][indx::period] = ds1['target_disparity'][indx::period]
+        ds1['target_disparity'][indx::period] = tmp
+
+        tmp = ds0['gt_ds'][indx::period].copy() 
+        ds0['gt_ds'][indx::period] = ds1['gt_ds'][indx::period]
+        ds1['gt_ds'][indx::period] = tmp
+    
+def shuffle_chunks_in_place(datasets_data, tiles_groups_per_chunk):
+    """
+    Improve shuffling by preserving indices inside batches (0 <->0, ... 39 <->39 for 40 tile group batches)
+    """
+    num_files = len(datasets_data)
+    #chunks_per_file = datasets_data[0]['target_disparity']
+    for nf, ds in enumerate(datasets_data):
+        groups_per_file = ds['corr2d'].shape[0]
+        chunks_per_file = groups_per_file//tiles_groups_per_chunk
+        permut = np.random.permutation(chunks_per_file)
+        ds['corr2d'] =           ds['corr2d'].          reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        ds['target_disparity'] = ds['target_disparity'].reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+        ds['gt_ds'] =            ds['gt_ds'].           reshape((chunks_per_file,-1))[permut].reshape((groups_per_file,-1))
+
+        
+def replaceDataset(datasets_data, new_dataset, nset, period, findx):
+    """
+    Replace one file in the dataset
+    """
+    datasets_data[findx]['corr2d']          [nset::period] =  new_dataset['corr2d'] 
+    datasets_data[findx]['target_disparity'][nset::period] =  new_dataset['target_disparity'] 
+    datasets_data[findx]['gt_ds']           [nset::period] =  new_dataset['gt_ds'] 
+       
+    
+
+def zip_lvar_hvar(datasets_all_data, del_src = True):
+#    cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+# Reformat input data
+    num_sets_to_combine = len(datasets_all_data)
+    datasets_data = []
+    if num_sets_to_combine:
+        for nrec in range(len(datasets_all_data[0])):
+            recs = [[] for _ in range(num_sets_to_combine)]
+            for nset, datasets in enumerate(datasets_all_data):
+                recs[nset] = datasets[nrec]
+                
+            rec = {'corr2d':           np.empty((recs[0]['corr2d'].shape[0]*num_sets_to_combine,          recs[0]['corr2d'].shape[1]),dtype=np.float32),
+                   'target_disparity': np.empty((recs[0]['target_disparity'].shape[0]*num_sets_to_combine,recs[0]['target_disparity'].shape[1]),dtype=np.float32),
+                   'gt_ds':            np.empty((recs[0]['gt_ds'].shape[0]*num_sets_to_combine,           recs[0]['gt_ds'].shape[1]),dtype=np.float32)}
+            
+            for nset, reci in enumerate(recs):
+                rec['corr2d']          [nset::num_sets_to_combine] =  recs[nset]['corr2d'] 
+                rec['target_disparity'][nset::num_sets_to_combine] =  recs[nset]['target_disparity'] 
+                rec['gt_ds']           [nset::num_sets_to_combine] =  recs[nset]['gt_ds'] 
+            if del_src:
+                for nset in range(num_sets_to_combine):
+                    datasets_all_data[nset][nrec] = None
+            datasets_data.append(rec)
+    return datasets_data    
+         
+
+# list of dictionaries  
+def reduce_tile_size(datasets_data, num_tile_layers, reduced_tile_side):
+    if (not datasets_data is None) and (len (datasets_data) > 0): 
+        tsz = (datasets_data[0]['corr2d'].shape[1])// num_tile_layers # 81 # list index out of range
+        tss = int(np.sqrt(tsz)+0.5)
+        offs = (tss - reduced_tile_side) // 2
+        for rec in datasets_data:
+            rec['corr2d'] =  (rec['corr2d'].reshape((-1, num_tile_layers,  tss, tss))
+                                 [..., offs:offs+reduced_tile_side, offs:offs+reduced_tile_side].
+                                 reshape(-1,num_tile_layers*reduced_tile_side*reduced_tile_side))
+
+def eval_results(rslt_path, absolute,
+                 min_disp =       -0.1, #minimal GT disparity
+                 max_disp =       20.0, # maximal GT disparity
+                 max_ofst_target = 1.0,
+                 max_ofst_result = 1.0,
+                 str_pow =         2.0,
+                 radius =          0):
+#    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in [
+    variants = [[         -0.1,         5.0,              0.5,            0.5,          1.0],           
+                [         -0.1,         5.0,              0.5,            0.5,          2.0],
+                [         -0.1,         5.0,              0.2,            0.2,          1.0],
+                [         -0.1,         5.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              0.5,            0.5,          1.0],           
+                [         -0.1,        20.0,              0.5,            0.5,          2.0],
+                [         -0.1,        20.0,              0.2,            0.2,          1.0],
+                [         -0.1,        20.0,              0.2,            0.2,          2.0],
+                [         -0.1,        20.0,              1.0,            1.0,          1.0],
+                [min_disp, max_disp, max_ofst_target, max_ofst_result, str_pow]]
+    
+
+    rslt = np.load(result_file)
+    not_nan  =  ~np.isnan(rslt[...,0])
+    not_nan &=  ~np.isnan(rslt[...,1])
+    not_nan &=  ~np.isnan(rslt[...,2])
+    not_nan &=  ~np.isnan(rslt[...,3])
+    not_nan_ext = np.zeros((rslt.shape[0] + 2*radius,rslt.shape[1] + 2 * radius),dtype=np.bool) 
+    not_nan_ext[radius:-radius,radius:-radius] = not_nan
+    for dy in range(2*radius+1):
+        for dx in range(2*radius+1):
+            not_nan_ext[dy:dy+not_nan.shape[0], dx:dx+not_nan.shape[1]] &= not_nan
+    not_nan = not_nan_ext[radius:-radius,radius:-radius]         
+        
+    if  not absolute:
+        rslt[...,0] +=  rslt[...,1]
+    nn_disparity =     np.nan_to_num(rslt[...,0], copy = False)
+    target_disparity = np.nan_to_num(rslt[...,1], copy = False)
+    gt_disparity =     np.nan_to_num(rslt[...,2], copy = False)
+    gt_strength =      np.nan_to_num(rslt[...,3], copy = False)
+    rslt = []
+    for min_disparity, max_disparity, max_offset_target, max_offset_result, strength_pow in variants:
+        good_tiles = not_nan.copy();
+        good_tiles &= (gt_disparity >= min_disparity)
+        good_tiles &= (gt_disparity <= max_disparity)
+        good_tiles &= (target_disparity != gt_disparity)
+        good_tiles &= (np.abs(target_disparity - gt_disparity) <= max_offset_target)
+        good_tiles &= (np.abs(target_disparity - nn_disparity) <= max_offset_result)
+        gt_w =  gt_strength * good_tiles
+        gt_w = np.power(gt_w,strength_pow)
+        sw = gt_w.sum() 
+        diff0 = target_disparity - gt_disparity
+        diff1 = nn_disparity -     gt_disparity
+        diff0_2w = gt_w*diff0*diff0
+        diff1_2w = gt_w*diff1*diff1
+        rms0 = np.sqrt(diff0_2w.sum()/sw) 
+        rms1 = np.sqrt(diff1_2w.sum()/sw)
+        print ("%7.3f<disp<%7.3f, offs_tgt<%5.2f, offs_rslt<%5.2f pwr=%05.3f, rms0=%7.4f, rms1=%7.4f (gain=%7.4f) num good tiles = %5d"%(
+            min_disparity, max_disparity, max_offset_target,  max_offset_result, strength_pow, rms0, rms1, rms0/rms1, good_tiles.sum() ))
+        rslt.append([rms0,rms1])
+    return rslt 
+    
+                         
+
+"""
+Start of the main code
+"""
+"""
+try:
+    train_filenameTFR =  sys.argv[1]
+except IndexError:
+    train_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_00.tfrecords"
+try:
+    test_filenameTFR =  sys.argv[2]
+except IndexError:
+    test_filenameTFR = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/test.tfrecords"
+#FILES_PER_SCENE
+train_filenameTFR1 = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/tf_data/train_01.tfrecords"
+"""
+data_dir =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" 
+data_dir1 = "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_2" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg" #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+ 
+img_dir =   "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_dbg"
+
+dir_train_lvar =  data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar =  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_lvar1 = data_dir1 #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_1" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_train_hvar1 = data_dir1 # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_lvar =  data_dir # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center/" # data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_test_hvar =  "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main_3" # data_dir #"/home/eyesis/x3d_data/data_sets/tf_data_5x5_center" #  data_dir # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main"
+dir_img =        os.path.join(img_dir,"img") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/img"
+dir_result =     os.path.join(data_dir,"result") # "/home/eyesis/x3d_data/data_sets/tf_data_5x5_main/result"
+
+files_train_lvar = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+files_train_lvar1 = ["train000_R2_LE_1.5.tfrecords",
+                    "train001_R2_LE_1.5.tfrecords",
+                    "train002_R2_LE_1.5.tfrecords",
+                    "train003_R2_LE_1.5.tfrecords",
+                    "train004_R2_LE_1.5.tfrecords",
+                    "train005_R2_LE_1.5.tfrecords",
+                    "train006_R2_LE_1.5.tfrecords",
+                    "train007_R2_LE_1.5.tfrecords",
+                    ]
+files_train_hvar1 = ["train000_R2_GT_1.5.tfrecords",
+                    "train001_R2_GT_1.5.tfrecords",
+                    "train002_R2_GT_1.5.tfrecords",
+                    "train003_R2_GT_1.5.tfrecords",
+                    "train004_R2_GT_1.5.tfrecords",
+                    "train005_R2_GT_1.5.tfrecords",
+                    "train006_R2_GT_1.5.tfrecords",
+                    "train007_R2_GT_1.5.tfrecords",
+]
+
+
+
+"""
+Try again - all hvar training, train than different hvar testing.
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-bothtrain-test-hvar" --port=7069
+Seems that even different (but used) hvar perfectly match each other, but training for both lvar and hvar never get
+match for either of hvar/lvar, even those that were used for training
+tensorboard --logdir="attic/nn_ds_neibs6_graph0-0RNS-19-tested_with_same_hvar_lvar_as_trained" --port=7070
+
+try same with higher LR - will they eventually converge?
+
+Compare with other (not used) train sets (use 7 of each instead of 8, 8-th as test)
+"""
+#just testing:
+#files_train_lvar = files_train_hvar
+files_test_lvar =  ["train004_R2_GT_1.5.tfrecords"]# ["train007_R2_LE_1.5.tfrecords"]# "testTEST_R2_LE_1.5.tfrecords"] # testTEST_R2_LE_1.5.tfrecords"]
+files_test_hvar =  ["train000_R2_GT_1.5.tfrecords"]#"testTEST_R2_GT_1.5.tfrecords"] # "testTEST_R2_GT_1.5.tfrecords"]
+#files_img =        ['1527257933_150165-v04'] # overlook
+#files_img =        ['1527256858_150165-v01'] # state Street
+files_img =        ['1527256816_150165-v02'] # staete Street
+for i, path in enumerate(files_train_lvar):
+    files_train_lvar[i]=os.path.join(dir_train_lvar, path)
+    
+for i, path in enumerate(files_train_hvar):
+    files_train_hvar[i]=os.path.join(dir_train_hvar, path)
+
+# Second set of files
+
+for i, path in enumerate(files_train_lvar1):
+    files_train_lvar1[i]=os.path.join(dir_train_lvar1, path)
+    
+for i, path in enumerate(files_train_hvar1):
+    files_train_hvar1[i]=os.path.join(dir_train_hvar1, path)
+
+
+    
+for i, path in enumerate(files_test_lvar):
+    files_test_lvar[i]=os.path.join(dir_test_lvar, path)
+    
+for i, path in enumerate(files_test_hvar):
+    files_test_hvar[i]=os.path.join(dir_test_hvar, path)
+
+result_files=[]
+for i, path in enumerate(files_img):
+    files_img[i] =      os.path.join(dir_img,    path+'.tfrecords')
+    result_files.append(os.path.join(dir_result, path+"_"+SUFFIX+'.npy'))
+
+
+#file_test_hvar=  None
+weight_hvar = 0.26
+weight_lvar = 1.0 - weight_hvar 
+
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+
+for result_file in result_files:
+    try:
+        eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)
+    #exit(0)
+    except:
+        pass
+
+datasets_img = []
+for fpath in files_img:
+    print_time("Importing test image data from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_img.append(       {"corr2d":           corr2d,
+                                "target_disparity": target_disparity,
+                                "gt_ds":            gt_ds})
+    print_time("  Done")
+gtruth =   datasets_img[0]['gt_ds'].copy()
+t_disp =   datasets_img[0]['target_disparity'].reshape([-1,1]).copy()
+extend_img_to_clusters(datasets_img, radius = CLUSTER_RADIUS)
+#reformat_to_clusters(datasets_img) already this format   
+replace_nan(datasets_img)
+pass
+pass
+
+
+datasets_train_lvar =  []
+datasets_train_hvar =  []
+datasets_train_lvar1 = []
+datasets_train_hvar1 = []
+for fpath in files_train_lvar:
+    print_time("Importing train data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+for fpath in files_train_hvar:
+    print_time("Importing train data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_train_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+if TWO_TRAINS:
+    for fpath in files_train_lvar1:
+        print_time("Importing train data1 (low variance) from "+fpath, end="")
+        corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+        datasets_train_lvar1.append({"corr2d":corr2d,
+                                     "target_disparity":target_disparity,
+                                     "gt_ds":gt_ds})
+        print_time("  Done")
+    for fpath in files_train_hvar1:
+        print_time("Importing train data1 (high variance) from "+fpath, end="")
+        corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+        datasets_train_hvar1.append({"corr2d":corr2d,
+                                     "target_disparity":target_disparity,
+                                     "gt_ds":gt_ds})
+        print_time("  Done")
+
+
+
+datasets_test_lvar = []
+for fpath in files_test_lvar:
+    print_time("Importing test data (low variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_lvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+
+datasets_test_hvar = []
+for fpath in files_test_hvar:
+    print_time("Importing test data (high variance) from "+fpath, end="")
+    corr2d, target_disparity, gt_ds = readTFRewcordsEpoch(fpath)
+    datasets_test_hvar.append({"corr2d":corr2d,
+                                "target_disparity":target_disparity,
+                                "gt_ds":gt_ds})
+    print_time("  Done")
+    
+    
+    
+    
+#CLUSTER_RADIUS
+cluster_size = (2 * CLUSTER_RADIUS + 1) * (2 * CLUSTER_RADIUS + 1)
+center_tile_index = 2 * CLUSTER_RADIUS * (CLUSTER_RADIUS + 1)
+# Reformat input data
+if FILE_TILE_SIDE > TILE_SIDE:
+    print_time("Reducing correlation tile size from %d to %d"%(FILE_TILE_SIDE, TILE_SIDE), end="")
+    reduce_tile_size(datasets_train_lvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar,  TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_lvar1, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_train_hvar1, TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_lvar,   TILE_LAYERS, TILE_SIDE)
+    reduce_tile_size(datasets_test_hvar,   TILE_LAYERS, TILE_SIDE)
+    print_time("  Done")
+pass
+print_time("Reshaping train data (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar)
+print_time("  Done")
+print_time("Reshaping train data (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar)
+print_time("  Done")
+
+print_time("Reshaping train data1 (low variance)", end="")
+reformat_to_clusters(datasets_train_lvar1)
+print_time("  Done")
+print_time("Reshaping train data1 (high variance)", end="")
+reformat_to_clusters(datasets_train_hvar1)
+print_time("  Done")
+
+print_time("Reshaping test data (low variance)", end="")
+reformat_to_clusters(datasets_test_lvar)
+print_time("  Done")
+print_time("Reshaping test data (high variance)", end="")
+reformat_to_clusters(datasets_test_hvar)
+print_time("  Done")
+pass    
+
+"""
+datasets_train_lvar & datasets_train_hvar ( that will increase batch size and placeholders twice
+test has to have even original, batches will not zip - just use two batches for one big one
+"""
+
+
+
+datasets_train_list =      [datasets_train_lvar,  datasets_train_hvar]
+if TWO_TRAINS:
+    datasets_train_list += [datasets_train_lvar1, datasets_train_hvar1]
+
+if ZIP_LHVAR:
+    print_time("Zipping together datasets datasets_train_lvar and datasets_train_hvar", end="")
+    datasets_train = zip_lvar_hvar(datasets_train_list, del_src = True) # no shuffle, delete src
+    print_time("  Done")
+else:
+    #Alternate lvar/hvar
+    datasets_train = []
+    datasets_weights_train = []
+    for indx in range(max(len(datasets_train_lvar),len(datasets_train_hvar))):
+        if (indx < len(datasets_train_lvar)):
+            datasets_train.append(datasets_train_lvar[indx])
+            datasets_weights_train.append(weight_lvar)
+        if (indx < len(datasets_train_hvar)):
+            datasets_train.append(datasets_train_hvar[indx])
+            datasets_weights_train.append(weight_hvar)
+
+datasets_test = []
+datasets_weights_test = []
+for dataset_test_lvar in datasets_test_lvar:
+    datasets_test.append(dataset_test_lvar)
+    datasets_weights_test.append(weight_lvar)
+for dataset_test_hvar in datasets_test_hvar:
+    datasets_test.append(dataset_test_hvar)
+    datasets_weights_test.append(weight_hvar)
+
+    
+corr2d_train_placeholder =           tf.placeholder(datasets_train[0]['corr2d'].dtype,           (None,FEATURES_PER_TILE * cluster_size)) # corr2d_train.shape)
+target_disparity_train_placeholder = tf.placeholder(datasets_train[0]['target_disparity'].dtype, (None,1 *   cluster_size))  #target_disparity_train.shape)
+gt_ds_train_placeholder =            tf.placeholder(datasets_train[0]['gt_ds'].dtype,            (None,2 *   cluster_size)) #gt_ds_train.shape)
+#dataset_tt    TensorSliceDataset: <TensorSliceDataset shapes: {gt_ds: (50,), corr2d: (8100,), target_disparity: (25,)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>    
+dataset_tt = tf.data.Dataset.from_tensor_slices({
+    "corr2d":corr2d_train_placeholder,
+    "target_disparity": target_disparity_train_placeholder,
+    "gt_ds": gt_ds_train_placeholder})
+
+
+#dataset_train_size = len(corr2d_train)
+
+#dataset_train_size = len(datasets_train_lvar[0]['corr2d'])
+dataset_train_size = len(datasets_train[0]['corr2d'])
+dataset_train_size //= BATCH_SIZE
+dataset_test_size = len(datasets_test_lvar[0]['corr2d'])
+dataset_test_size //= BATCH_SIZE
+dataset_img_size = len(datasets_img[0]['corr2d'])
+dataset_img_size //= BATCH_SIZE
+
+#print_time("dataset_tt.output_types "+str(dataset_train.output_types)+", dataset_train.output_shapes "+str(dataset_train.output_shapes)+", number of elements="+str(dataset_train_size))
+dataset_tt = dataset_tt.batch(BATCH_SIZE)
+dataset_tt = dataset_tt.prefetch(BATCH_SIZE)
+iterator_tt = dataset_tt.make_initializable_iterator()
+next_element_tt = iterator_tt.get_next()
+#print("dataset_tt.output_types "+str(dataset_tt.output_types)+", dataset_tt.output_shapes "+str(dataset_tt.output_shapes)+", number of elements="+str(dataset_train_size))
+#BatchDataset: <BatchDataset shapes: {gt_ds: (?, 50), corr2d: (?, 8100), target_disparity: (?, 25)}, types: {gt_ds: tf.float32, corr2d: tf.float32, target_disparity: tf.float32}>
+"""
+next_element_tt    dict: {'gt_ds': <tf.Tensor 'IteratorGetNext:1' shape=(?, 50) dtype=float32>, 'corr2d': <tf.Tensor 'IteratorGetNext:0' shape=(?, 8100) dtype=float32>, 'target_disparity': <tf.Tensor 'IteratorGetNext:2' shape=(?, 25) dtype=float32>}    
+    'corr2d' (140405473715624)    Tensor: Tensor("IteratorGetNext:0", shape=(?, 8100), dtype=float32)    
+    'gt_ds' (140405473715680)    Tensor: Tensor("IteratorGetNext:1", shape=(?, 50), dtype=float32)    
+    'target_disparity' (140405501995888)    Tensor: Tensor("IteratorGetNext:2", shape=(?, 25), dtype=float32)    
+"""
+
+#https://www.tensorflow.org/versions/r1.5/programmers_guide/datasets
+result_dir = './attic/result_neibs_'+     SUFFIX+'/'
+checkpoint_dir = './attic/result_neibs_'+ SUFFIX+'/'
+save_freq = 500
+def lrelu(x):
+    return tf.maximum(x*0.2,x)
+#    return tf.nn.relu(x)
+
+def sym_inputs8(inp):
+    """
+    get input vector [?:4*9*9+1] (last being target_disparity) and reorder for horizontal flip,
+    vertical flip and transpose (8 variants, mode + 1 - hor, +2 - vert, +4 - transpose)
+    return same lengh, reordered
+    """
+    with tf.name_scope("sym_inputs8"):
+        td =           inp[:,-1:] # tf.reshape(inp,[-1], name = "td")[-1]
+        inp_corr =     tf.reshape(inp[:,:-1],[-1,4,TILE_SIDE,TILE_SIDE], name = "inp_corr")
+        inp_corr_h =   tf.stack([-inp_corr  [:,0,:,-1::-1], inp_corr  [:,1,:,-1::-1], -inp_corr  [:,3,:,-1::-1], -inp_corr  [:,2,:,-1::-1]], axis=1, name = "inp_corr_h")
+        inp_corr_v =   tf.stack([ inp_corr  [:,0,-1::-1,:],-inp_corr  [:,1,-1::-1,:],  inp_corr  [:,3,-1::-1,:],  inp_corr  [:,2,-1::-1,:]], axis=1, name = "inp_corr_v")
+        inp_corr_hv =  tf.stack([ inp_corr_h[:,0,-1::-1,:],-inp_corr_h[:,1,-1::-1,:],  inp_corr_h[:,3,-1::-1,:],  inp_corr_h[:,2,-1::-1,:]], axis=1, name = "inp_corr_hv")
+        inp_corr_t =   tf.stack([tf.transpose(inp_corr   [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr   [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr   [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_t")
+        inp_corr_ht =  tf.stack([tf.transpose(inp_corr_h [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_h [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_h [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_ht")
+        inp_corr_vt =  tf.stack([tf.transpose(inp_corr_v [:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_v [:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_v [:,3], perm=[0,2,1])], axis=1, name = "inp_corr_vt")
+        inp_corr_hvt = tf.stack([tf.transpose(inp_corr_hv[:,1], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,0], perm=[0,2,1]),
+                                 tf.transpose(inp_corr_hv[:,2], perm=[0,2,1]),
+                                -tf.transpose(inp_corr_hv[:,3], perm=[0,2,1])], axis=1, name = "inp_corr_hvt")
+#        return td, [inp_corr, inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+        """
+        return [tf.concat([tf.reshape(inp_corr,    [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[inp_corr.shape[0],-1]),td], axis=1,name = "out_corr_hvt")]
+        """
+        cl = 4 * TILE_SIDE * TILE_SIDE
+        return [tf.concat([tf.reshape(inp_corr,    [-1,cl]),td], axis=1,name = "out_corr"),
+                tf.concat([tf.reshape(inp_corr_h,  [-1,cl]),td], axis=1,name = "out_corr_h"),
+                tf.concat([tf.reshape(inp_corr_v,  [-1,cl]),td], axis=1,name = "out_corr_v"),
+                tf.concat([tf.reshape(inp_corr_hv, [-1,cl]),td], axis=1,name = "out_corr_hv"),
+                tf.concat([tf.reshape(inp_corr_t,  [-1,cl]),td], axis=1,name = "out_corr_t"),
+                tf.concat([tf.reshape(inp_corr_ht, [-1,cl]),td], axis=1,name = "out_corr_ht"),
+                tf.concat([tf.reshape(inp_corr_vt, [-1,cl]),td], axis=1,name = "out_corr_vt"),
+                tf.concat([tf.reshape(inp_corr_hvt,[-1,cl]),td], axis=1,name = "out_corr_hvt")]
+#                           inp_corr_h, inp_corr_v, inp_corr_hv, inp_corr_t, inp_corr_ht, inp_corr_vt, inp_corr_hvt]
+    
+
+def network_sub(input, layout, reuse, sym8 = False):
+    last_indx = None;
+    fc = []
+    inp_weights = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+               fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_sub'+str(i), reuse = reuse))
+           else:
+               inp = input
+               if sym8:
+                   inp8 = sym_inputs8(inp)
+                   num_non_sum = num_outs %  len(inp8) # if number of first layer outputs is not multiple of 8
+                   num_sym8 =    num_outs // len(inp8) # number of symmetrical groups
+                   fc_sym = []
+                   for j in range (len(inp8)): # ==8
+                       reuse_this = reuse | (j > 0)
+                       scp = 'g_fc_sub'+str(i)
+                       fc_sym.append(slim.fully_connected(inp8[j],    num_sym8, activation_fn=lrelu, scope= scp,     reuse = reuse_this))
+                       if not reuse_this:
+                           with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
+                              inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs])) 
+                   if num_non_sum > 0:
+                       reuse_this = reuse
+                       scp = 'g_fc_sub'+str(i)+"r"
+                       fc_sym.append(slim.fully_connected(inp,     num_non_sum, activation_fn=lrelu, scope=scp, reuse = reuse_this))    
+                       if not reuse_this:
+                           with tf.variable_scope(scp,reuse=True) : # tf.AUTO_REUSE):
+                              inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs])) 
+                   fc.append(tf.concat(fc_sym, 1, name='sym_input_layer'))
+               else:
+                   scp = 'g_fc_sub'+str(i)
+                   fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope= scp, reuse = reuse))
+                   if not reuse:
+                       with tf.variable_scope(scp, reuse=True) : # tf.AUTO_REUSE):
+                          inp_weights.append(tf.get_variable('weights')) # ,shape=[inp.shape[1],num_outs])) 
+           
+    return fc[-1], inp_weights
+
+def network_inter(input, layout):
+    last_indx = None;
+    fc = []
+    for i, num_outs in enumerate (layout):
+        if num_outs:
+           if fc:
+               inp = fc[-1]
+           else:
+               inp = input
+           fc.append(slim.fully_connected(inp,    num_outs, activation_fn=lrelu, scope='g_fc_inter'+str(i)))
+    if USE_CONFIDENCE:
+        fc_out  = slim.fully_connected(fc[-1],     2, activation_fn=lrelu,scope='g_fc_inter_out')
+    else:     
+        fc_out  = slim.fully_connected(fc[-1],     1, activation_fn=None,scope='g_fc_inter_out')
+        #If using residual disparity, split last layer into 2 or remove activation and add rectifier to confidence only  
+    return fc_out
+
+def network_siam(input, # now [?,9,325]-> [?,25,325]
+                 layout1, 
+                 layout2,
+                 sym8 =        False,
+                 only_tile =   None): # just for debugging - feed only data from the center sub-network
+    with tf.name_scope("Siam_net"):
+        inp_weights = []
+        num_legs =  input.shape[1] # == 9
+        inter_list = []
+        reuse = False
+        for i in range (num_legs):
+            if (only_tile is None) or (i == only_tile):
+#                inter_list.append(network_sub(input[:,i,:],
+#                                          layout= layout1,
+#                                          reuse= reuse,
+#                                          sym8 = sym8))
+                ns, ns_weights = network_sub(input[:,i,:],
+                                          layout= layout1,
+                                          reuse= reuse,
+                                          sym8 = sym8)
+                inter_list.append(ns)
+                inp_weights += ns_weights
+                reuse = True
+        inter_tensor = tf.concat(inter_list, 1, name='inter_tensor')
+        return  network_inter (inter_tensor, layout2),  inp_weights 
+
+def debug_gt_variance(
+        indx,        # This tile index (0..8)
+        center_indx, # center tile index
+        gt_ds_batch # [?:9:2]
+        ):
+    with tf.name_scope("Debug_GT_Variance"):
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        d_gt_this =   tf.reshape(gt_ds_batch[:,2 * indx],[-1],                     name = "d_this")
+        d_gt_center = tf.reshape(gt_ds_batch[:,2 * center_indx],[-1],              name = "d_center")
+        d_gt_diff =   tf.subtract(d_gt_this, d_gt_center,                          name = "d_diff")
+        d_gt_diff2 =  tf.multiply(d_gt_diff, d_gt_diff,                            name = "d_diff2")
+        d_gt_var =    tf.reduce_mean(d_gt_diff2,                                   name = "d_gt_var")
+        return  d_gt_var
+    
+    
+def batchLoss(out_batch,                   # [batch_size,(1..2)] tf_result
+              target_disparity_batch,      # [batch_size]        tf placeholder
+              gt_ds_batch,                 # [batch_size,2]      tf placeholder
+              absolute_disparity =     True, #when false there should be no activation on disparity output ! 
+              use_confidence =         True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            1.0,
+              error2_offset =          0.0025, # 0.0, # 0.0025, # (0.05^2) ~= coring
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False):  # use calculated disparity for disparity weight boosting (False - use target disparity)
+               
+    with tf.name_scope("BatchLoss"):
+        """
+        Here confidence should be after relU. Disparity - may be also if absolute, but no activation if output is residual disparity
+        """
+        tf_lambda_conf_avg = tf.constant(lambda_conf_avg, dtype=tf.float32, name="tf_lambda_conf_avg")
+        tf_lambda_conf_pwr = tf.constant(lambda_conf_pwr, dtype=tf.float32, name="tf_lambda_conf_pwr")
+        tf_conf_pwr =        tf.constant(conf_pwr,        dtype=tf.float32, name="tf_conf_pwr")
+        tf_gt_conf_offset =  tf.constant(gt_conf_offset,  dtype=tf.float32, name="tf_gt_conf_offset")
+        tf_gt_conf_pwr =     tf.constant(gt_conf_pwr,     dtype=tf.float32, name="tf_gt_conf_pwr")
+        tf_num_tiles =       tf.shape(gt_ds_batch)[0]
+        tf_0f =              tf.constant(0.0,             dtype=tf.float32, name="tf_0f")
+        tf_1f =              tf.constant(1.0,             dtype=tf.float32, name="tf_1f")
+        tf_maxw =            tf.constant(1.0,             dtype=tf.float32, name="tf_maxw")
+        if gt_conf_pwr == 0:
+            w = tf.ones((out_batch.shape[0]), dtype=tf.float32,name="w_ones")
+        else:
+    #        w_slice = tf.slice(gt_ds_batch,[0,1],[-1,1],              name = "w_gt_slice")
+            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+            
+            w_sub =   tf.subtract      (w_slice, tf_gt_conf_offset,         name = "w_sub")
+    #        w_clip =  tf.clip_by_value(w_sub, tf_0f,tf_maxw,              name = "w_clip")
+            w_clip =  tf.maximum(w_sub, tf_0f,                              name = "w_clip")
+            if gt_conf_pwr == 1.0:
+                w = w_clip
+            else:
+                w=tf.pow(w_clip, tf_gt_conf_pwr, name = "w_pow")
+    
+        if use_confidence:
+            tf_num_tilesf =      tf.cast(tf_num_tiles, dtype=tf.float32,     name="tf_num_tilesf")
+    #        conf_slice =     tf.slice(out_batch,[0,1],[-1,1],                name = "conf_slice")
+            conf_slice =     tf.reshape(out_batch[:,1],[-1],                 name = "conf_slice")
+            conf_sum =       tf.reduce_sum(conf_slice,                       name = "conf_sum")
+            conf_avg =       tf.divide(conf_sum, tf_num_tilesf,              name = "conf_avg")
+            conf_avg1 =      tf.subtract(conf_avg, tf_1f,                    name = "conf_avg1")
+            conf_avg2 =      tf.square(conf_avg1,                            name = "conf_avg2")
+            cost2 =          tf.multiply (conf_avg2, tf_lambda_conf_avg,     name = "cost2")
+    
+            iconf_avg =      tf.divide(tf_1f, conf_avg,                      name = "iconf_avg")
+            nconf =          tf.multiply (conf_slice, iconf_avg,             name = "nconf") #normalized confidence
+            nconf_pwr =      tf.pow(nconf, conf_pwr,                         name = "nconf_pwr")
+            nconf_pwr_sum =  tf.reduce_sum(nconf_pwr,                        name = "nconf_pwr_sum")
+            nconf_pwr_offs = tf.subtract(nconf_pwr_sum, tf_1f,               name = "nconf_pwr_offs")
+            cost3 =          tf.multiply (conf_avg2, nconf_pwr_offs,         name = "cost3")
+            w_all =          tf.multiply (w, nconf,                          name = "w_all")
+        else:
+            w_all = w
+#            cost2 = 0.0
+#            cost3 = 0.0    
+        # normalize weights
+        w_sum =              tf.reduce_sum(w_all,                            name = "w_sum")
+        iw_sum =             tf.divide(tf_1f, w_sum,                         name = "iw_sum")
+        w_norm =             tf.multiply (w_all, iw_sum,                     name = "w_norm")
+        
+        disp_slice =         tf.reshape(out_batch[:,0],[-1],                 name = "disp_slice")
+        d_gt_slice =         tf.reshape(gt_ds_batch[:,0],[-1],               name = "d_gt_slice")
+        
+        td_flat =        tf.reshape(target_disparity_batch,[-1],         name = "td_flat")
+        if absolute_disparity:
+            adisp =          disp_slice
+        else:
+            adisp =          tf.add(disp_slice, td_flat,                     name = "adisp")
+        out_diff =           tf.subtract(adisp, d_gt_slice,                  name = "out_diff")
+            
+            
+        out_diff2 =          tf.square(out_diff,                             name = "out_diff2")
+        out_wdiff2 =         tf.multiply (out_diff2, w_norm,                 name = "out_wdiff2")
+        
+        cost1 =              tf.reduce_sum(out_wdiff2,                       name = "cost1")
+        
+        out_diff2_offset =   tf.subtract(out_diff2, error2_offset,           name = "out_diff2_offset")
+        out_diff2_biased =   tf.maximum(out_diff2_offset, 0.0,               name = "out_diff2_biased")
+        
+        # calculate disparity-based weight boost
+        if use_out:
+            dispw =          tf.clip_by_value(adisp, disp_wmin, disp_wmax,   name = "dispw")
+        else:
+            dispw =          tf.clip_by_value(td_flat, disp_wmin, disp_wmax, name = "dispw")
+        dispw_boost =        tf.divide(disp_wmax, dispw,                     name = "dispw_boost")
+        dispw_comp =         tf.multiply (dispw_boost, w_norm,               name = "dispw_comp") #HERE??
+#        dispw_comp =         tf.multiply (w_norm, w_norm,               name = "dispw_comp") #HERE??
+        dispw_sum =          tf.reduce_sum(dispw_comp,                       name = "dispw_sum")
+        idispw_sum =         tf.divide(tf_1f, dispw_sum,                     name = "idispw_sum")
+        dispw_norm =         tf.multiply (dispw_comp, idispw_sum,            name = "dispw_norm")
+        
+        out_diff2_wbiased =  tf.multiply(out_diff2_biased, dispw_norm,       name = "out_diff2_wbiased")
+#        out_diff2_wbiased =  tf.multiply(out_diff2_biased, w_norm,       name = "out_diff2_wbiased")
+        cost1b =             tf.reduce_sum(out_diff2_wbiased,                name = "cost1b")
+        
+        if use_confidence:
+            cost12 =         tf.add(cost1b, cost2,                           name = "cost12")
+            cost123 =        tf.add(cost12, cost3,                           name = "cost123")    
+            
+            return cost123, disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        else:
+            return cost1b,  disp_slice, d_gt_slice, out_diff,out_diff2, w_norm, out_wdiff2, cost1
+        
+        
+def weightsLoss(inp_weights):       # [batch_size,(1..2)] tf_result
+#                weights_lambdas):  # single lambda or same length as inp_weights.shape[1]
+    """
+    Enforcing 'smooth' weights for the input 2d correlation tiles
+    @return mean squared difference for each weight and average of 8 neighbors divided by mean squared weights
+    """
+    weight_ortho = 1.0
+    weight_diag  = 0.7
+    sw = 4.0 * (weight_ortho + weight_diag)
+    weight_ortho /= sw
+    weight_diag /=  sw
+#    w_neib = tf.const([[weight_diag,  weight_ortho, weight_diag],
+#                       [weight_ortho, -1.0,         weight_ortho],
+#                       [weight_diag,  weight_ortho, weight_diag]])
+    with tf.name_scope("WeightsLoss"):
+        # Adding 1 tile border
+        tf_inp =     tf.reshape(inp_weights[:TILE_LAYERS * TILE_SIZE,:], [TILE_LAYERS, FILE_TILE_SIDE, FILE_TILE_SIDE, inp_weights.shape[1]], name = "tf_inp")
+        tf_inp_ext_h = tf.concat([tf_inp       [:, :,  :1, :], tf_inp,       tf_inp      [:,   :, -1:, :]], axis = 2, name ="tf_inp_ext_h")
+        tf_inp_ext   = tf.concat([tf_inp_ext_h [:, :1, :,  :], tf_inp_ext_h, tf_inp_ext_h[:, -1:,   :, :]], axis = 1, name ="tf_inp_ext")
+        s_ortho = tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:,:] + tf_inp_ext[:,1:-1,:-2,:] + tf_inp_ext[:,1:-1, 2:, :] 
+        s_corn =  tf_inp_ext[:, :-2,:-2,:] + tf_inp_ext[:, :-2, 2:,:] + tf_inp_ext[:,2:,  :-2,:] + tf_inp_ext[:,2:  , 2:, :]
+        w_diff =  tf.subtract(tf_inp, s_ortho * weight_ortho + s_corn * weight_diag, name="w_diff") 
+        w_diff2 = tf.multiply(w_diff, w_diff,                                        name="w_diff2") 
+        w_var =   tf.reduce_mean(w_diff2,                                            name="w_var")
+        w2_mean = tf.reduce_mean(inp_weights * inp_weights,                          name="w2_mean")
+        w_rel =   tf.divide(w_var, w2_mean,                                          name= "w_rel")
+        return w_rel # scalar, cost for weights non-smoothness in 2d
+        
+        
+    
+corr2d_Nx325 = tf.concat([tf.reshape(next_element_tt['corr2d'],[-1,cluster_size,FEATURES_PER_TILE], name="coor2d_cluster"),
+                          tf.reshape(next_element_tt['target_disparity'], [-1,cluster_size, 1], name="targdisp_cluster")], axis=2, name = "corr2d_Nx325")
+
+out, inp_weights =       network_siam(input=corr2d_Nx325,
+                                      layout1 =   NN_LAYOUT1, 
+                                      layout2 =   NN_LAYOUT2,
+                                      sym8 =      SYM8_SUB,
+                                      only_tile = ONLY_TILE) #Remove/put None for normal operation
+#            w_slice = tf.reshape(gt_ds_batch[:,1],[-1],                     name = "w_gt_slice")
+
+# Extract target disparity and GT corresponding to the center tile (reshape - just to name)
+#target_disparity_batch_center = tf.reshape(next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1] , [-1,1],  name = "target_center")
+#gt_ds_batch_center =            tf.reshape(next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * center_tile_index+1],    [-1,2],  name = "gt_ds_center")
+
+G_loss, _disp_slice, _d_gt_slice, _out_diff, _out_diff2, _w_norm, _out_wdiff2, _cost1 = batchLoss(out_batch =         out,        # [batch_size,(1..2)] tf_result
+              target_disparity_batch=  next_element_tt['target_disparity'][:,center_tile_index:center_tile_index+1], # target_disparity_batch_center, # next_element_tt['target_disparity'], # target_disparity, ### target_d,   # [batch_size]        tf placeholder
+              gt_ds_batch =            next_element_tt['gt_ds'][:,2 * center_tile_index: 2 * (center_tile_index +1)],  # gt_ds_batch_center, ## next_element_tt['gt_ds'], # gt_ds, ### gt,         # [batch_size,2]      tf placeholder
+              absolute_disparity =     ABSOLUTE_DISPARITY,
+              use_confidence =         USE_CONFIDENCE, # True, 
+              lambda_conf_avg =        0.01,
+              lambda_conf_pwr =        0.1,
+              conf_pwr =               2.0,
+              gt_conf_offset =         0.08,
+              gt_conf_pwr =            2.0,
+              error2_offset =          0, # 0.0025, # (0.05^2)
+              disp_wmin =              1.0,    # minimal disparity to apply weight boosting for small disparities
+              disp_wmax =              8.0,    # maximal disparity to apply weight boosting for small disparities
+              use_out =                False)  # use calculated disparity for disparity weight boosting (False - use target disparity)
+
+if WLOSS_LAMBDA > 0.0:   
+    W_loss =     weightsLoss(inp_weights[0]) #    inp_weights - list of tensors, currently - just [0]
+    GW_loss =    tf.add(G_loss, WLOSS_LAMBDA * W_loss, name = "GW_loss")
+else:
+    GW_loss = G_loss
+    W_loss =     0.0
+#debug
+GT_variance =  debug_gt_variance(indx = 0,        # This tile index (0..8)
+                                 center_indx = 4, # center tile index
+                                 gt_ds_batch = next_element_tt['gt_ds'])# [?:18]
+              
+tf_ph_G_loss =  tf.placeholder(tf.float32,shape=None,name='G_loss_avg')
+tf_ph_W_loss =  tf.placeholder(tf.float32,shape=None,name='W_loss_avg')
+tf_ph_GW_loss = tf.placeholder(tf.float32,shape=None,name='GW_loss_avg')
+tf_ph_sq_diff = tf.placeholder(tf.float32,shape=None,name='sq_diff_avg')
+tf_gtvar_diff = tf.placeholder(tf.float32,shape=None,name='gtvar_diff')
+tf_img_test0 =  tf.placeholder(tf.float32,shape=None,name='img_test0')
+tf_img_test9 =  tf.placeholder(tf.float32,shape=None,name='img_test9')
+with tf.name_scope('sample'):
+    tf.summary.scalar("GW_loss",      GW_loss)
+    tf.summary.scalar("G_loss",       G_loss)
+    tf.summary.scalar("W_loss",       W_loss)
+    tf.summary.scalar("sq_diff",      _cost1)
+    tf.summary.scalar("gtvar_diff",   GT_variance)
+    
+with tf.name_scope('epoch_average'):
+    tf.summary.scalar("GW_loss_epoch", tf_ph_GW_loss)
+    tf.summary.scalar("G_loss_epoch",  tf_ph_G_loss)
+    tf.summary.scalar("W_loss_epoch",  tf_ph_W_loss)
+    tf.summary.scalar("sq_diff_epoch", tf_ph_sq_diff)
+    tf.summary.scalar("gtvar_diff",    tf_gtvar_diff)
+    
+    tf.summary.scalar("img_test0",     tf_img_test0)
+    tf.summary.scalar("img_test9",     tf_img_test9)
+
+t_vars=tf.trainable_variables()
+lr=tf.placeholder(tf.float32)
+G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(GW_loss)
+
+saver=tf.train.Saver()
+
+ROOT_PATH  = './attic/nn_ds_neibs8_graph'+SUFFIX+"/"
+TRAIN_PATH =  ROOT_PATH + 'train'
+TEST_PATH  =  ROOT_PATH + 'test'
+TEST_PATH1  = ROOT_PATH + 'test1'
+
+# CLEAN OLD STAFF
+shutil.rmtree(TRAIN_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH, ignore_errors=True)
+shutil.rmtree(TEST_PATH1, ignore_errors=True)
+
+WIDTH=324
+HEIGHT=242
+
+with tf.Session()  as sess:
+    
+    sess.run(tf.global_variables_initializer())
+    sess.run(tf.local_variables_initializer())
+    
+    merged = tf.summary.merge_all()
+    train_writer =    tf.summary.FileWriter(TRAIN_PATH, sess.graph)
+    test_writer  =    tf.summary.FileWriter(TEST_PATH, sess.graph)
+    test_writer1  =   tf.summary.FileWriter(TEST_PATH1, sess.graph)
+    
+    loss_gw_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    loss_g_train_hist=   np.empty(dataset_train_size, dtype=np.float32)
+    loss_w_train_hist=   np.empty(dataset_train_size, dtype=np.float32)
+    
+    loss_gw_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    loss_g_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    loss_w_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    
+    loss2_train_hist= np.empty(dataset_train_size, dtype=np.float32)
+    loss2_test_hist=  np.empty(dataset_test_size, dtype=np.float32)
+    train_gw_avg = 0.0
+    train_g_avg =  0.0
+    train_w_avg =  0.0
+    test_gw_avg =  0.0     
+    test_g_avg =   0.0     
+    test_w_avg =   0.0     
+
+    train2_avg = 0.0
+    test2_avg = 0.0
+    gtvar_train_hist=  np.empty(dataset_train_size, dtype=np.float32)
+    gtvar_test_hist=   np.empty(dataset_test_size, dtype=np.float32)
+    gtvar_train = 0.0
+    gtvar_test = 0.0
+    gtvar_train_avg = 0.0
+    gtvar_test_avg =  0.0
+    img_gain_test0 =  1.0
+    img_gain_test9 =  1.0
+    
+    num_train_variants = len(datasets_train)
+    for epoch in range (EPOCHS_TO_RUN):
+#        file_index = (epoch // 20) % 2 
+        file_index = epoch  % num_train_variants
+        if   epoch >=600:
+            learning_rate = LR600
+        elif epoch >=400:
+            learning_rate = LR400
+        elif epoch >=200:
+            learning_rate = LR200
+        elif epoch >=100:
+            learning_rate = LR100
+        else:
+            learning_rate = LR
+#        print ("sr1",file=sys.stderr,end=" ")
+        if (file_index == 0) and SHUFFLE_FILES:
+            num_sets = len(datasets_train_list)
+            print_time("Shuffling how datasets datasets_train_lvar and datasets_train_hvar are zipped together", end="")
+            for i in range(num_sets):
+                shuffle_in_place (datasets_train, i, num_sets)
+            print_time("  Done")
+            print_time("Shuffling tile chunks ", end="")
+            shuffle_chunks_in_place (datasets_train, 1)
+            print_time("  Done")
+            
+        sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:           datasets_train[file_index]['corr2d'],
+                                                     target_disparity_train_placeholder: datasets_train[file_index]['target_disparity'],
+                                                     gt_ds_train_placeholder:            datasets_train[file_index]['gt_ds']})
+        for i in range(dataset_train_size):
+            try:
+                train_summary,_, GW_loss_trained,  G_loss_trained,  W_loss_trained,  output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance  = sess.run(
+                    [   merged,
+                        G_opt,
+                        GW_loss,
+                        G_loss,
+                        W_loss,
+                        out,
+                        _disp_slice,
+                        _d_gt_slice,
+                        _out_diff,
+                        _out_diff2,
+                        _w_norm,
+                        _out_wdiff2,
+                        _cost1,
+                        GT_variance
+                    ],
+                    feed_dict={lr:            learning_rate,
+                               tf_ph_GW_loss: train_gw_avg,
+                               tf_ph_G_loss:  train_g_avg,
+                               tf_ph_W_loss:  train_w_avg,
+                               tf_ph_sq_diff: train2_avg,
+                               tf_gtvar_diff: gtvar_train_avg,
+                               tf_img_test0:  img_gain_test0,
+                               tf_img_test9:img_gain_test9}) # previous value of *_avg
+                
+                loss_gw_train_hist[i] = GW_loss_trained
+                loss_g_train_hist[i] =  G_loss_trained
+                loss_w_train_hist[i] =  W_loss_trained
+                loss2_train_hist[i] = out_cost1
+                gtvar_train_hist[i] = gt_variance
+            except tf.errors.OutOfRangeError:
+                print("train done at step %d"%(i))
+                break
+
+        train_gw_avg =      np.average(loss_gw_train_hist).astype(np.float32)     
+        train_g_avg =       np.average(loss_g_train_hist).astype(np.float32)     
+        train_w_avg =       np.average(loss_w_train_hist).astype(np.float32)     
+        train2_avg =      np.average(loss2_train_hist).astype(np.float32)
+        gtvar_train_avg = np.average(gtvar_train_hist).astype(np.float32)
+        
+        test_summaries = [0.0]*len(datasets_test)
+        tst_avg =        [0.0]*len(datasets_test)
+        tst2_avg =       [0.0]*len(datasets_test)
+        for ntest,dataset_test in enumerate(datasets_test):
+            sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_test['corr2d'],
+                                                    target_disparity_train_placeholder: dataset_test['target_disparity'],
+                                                    gt_ds_train_placeholder:            dataset_test['gt_ds']})
+            for i in range(dataset_test_size):
+                try:
+                    test_summaries[ntest], GW_loss_tested, G_loss_tested, W_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                        [merged,
+                         GW_loss,
+                         G_loss,
+                         W_loss,
+                         out,
+                         _disp_slice,
+                         _d_gt_slice,
+                         _out_diff,
+                         _out_diff2,
+                         _w_norm,
+                         _out_wdiff2,
+                         _cost1,
+                         GT_variance
+                         ],
+                         feed_dict={lr:            learning_rate,
+                                    tf_ph_GW_loss:  test_gw_avg,
+                                    tf_ph_G_loss:   test_g_avg,
+                                    tf_ph_W_loss:   test_w_avg,
+                                    tf_ph_sq_diff:  test2_avg,
+                                    tf_gtvar_diff:  gtvar_test_avg,
+                                    tf_img_test0:   img_gain_test0,
+                                    tf_img_test9:   img_gain_test9})  # previous value of *_avg
+                    loss_gw_test_hist[i] =  GW_loss_tested
+                    loss_g_test_hist[i] =   G_loss_tested
+                    loss_w_test_hist[i] =   W_loss_tested
+                    loss2_test_hist[i] = out_cost1
+                    gtvar_test_hist[i] = gt_variance
+                except tf.errors.OutOfRangeError:
+                    print("test done at step %d"%(i))
+                    break
+                    
+            test_gw_avg =  np.average(loss_gw_test_hist).astype(np.float32)
+            test_g_avg =  np.average(loss_g_test_hist).astype(np.float32)
+            test_w_avg =  np.average(loss_w_test_hist).astype(np.float32)
+            tst_avg[ntest] =  test_gw_avg   
+            test2_avg = np.average(loss2_test_hist).astype(np.float32)
+            tst2_avg[ntest] =  test2_avg   
+            gtvar_test_avg = np.average(gtvar_test_hist).astype(np.float32)
+             
+        train_writer.add_summary(train_summary, epoch)
+        test_writer.add_summary(test_summaries[0], epoch)
+        test_writer1.add_summary(test_summaries[1], epoch)
+        
+        print_time("%d:%d -> %f %f %f (%f %f %f) dbg:%f %f"%(epoch,i,train_gw_avg, tst_avg[0], tst_avg[1], train2_avg, tst2_avg[0], tst2_avg[1], gtvar_train_avg, gtvar_test_avg))
+        if ((epoch + 1) == EPOCHS_TO_RUN) or (((epoch + 1) % EPOCHS_FULL_TEST) == 0):
+###################################################
+# Read the full image
+################################################### 
+            test_summaries_img = [0.0]*len(datasets_img)
+        #    disp_out=  np.empty((dataset_img_size * BATCH_SIZE), dtype=np.float32)
+            disp_out=  np.empty((WIDTH*HEIGHT), dtype=np.float32)
+            
+            for ntest,dataset_img in enumerate(datasets_img):
+                sess.run(iterator_tt.initializer, feed_dict={corr2d_train_placeholder:      dataset_img['corr2d'],
+                                                        target_disparity_train_placeholder: dataset_img['target_disparity'],
+                                                        gt_ds_train_placeholder:            dataset_img['gt_ds']})
+                for start_offs in range(0,disp_out.shape[0],BATCH_SIZE):
+                    end_offs = min(start_offs+BATCH_SIZE,disp_out.shape[0])
+                    
+                    try:
+                        test_summaries_img[ntest], G_loss_tested, output, disp_slice, d_gt_slice, out_diff, out_diff2, w_norm, out_wdiff2, out_cost1, gt_variance = sess.run(
+                            [merged,
+                             G_loss,
+                             out,
+                             _disp_slice,
+                             _d_gt_slice,
+                             _out_diff,
+                             _out_diff2,
+                             _w_norm,
+                             _out_wdiff2,
+                             _cost1,
+                             GT_variance
+                             ],
+                             feed_dict={lr:            learning_rate,
+                                        tf_ph_GW_loss:  test_gw_avg,
+                                        tf_ph_G_loss:   test_g_avg,
+                                        tf_ph_W_loss:   test_w_avg,
+                                        tf_ph_sq_diff: test2_avg,
+                                        tf_gtvar_diff: gtvar_test_avg,
+                                        tf_img_test0:  img_gain_test0,
+                                        tf_img_test9:  img_gain_test9})  # previous value of *_avg
+                    except tf.errors.OutOfRangeError:
+                        print("test done at step %d"%(i))
+                        break
+                    try:
+                        disp_out[start_offs:end_offs] = output.flatten()
+                    except ValueError:
+                        print("dataset_img_size= %d, i=%d, output.shape[0]=%d "%(dataset_img_size, i, output.shape[0]))
+                        break;    
+                    pass
+                result_file = result_files[ntest]
+                try:
+                    os.makedirs(os.path.dirname(result_file))
+                except:
+                    pass     
+
+                rslt = np.concatenate([disp_out.reshape(-1,1), t_disp, gtruth],1)
+                np.save(result_file,           rslt.reshape(HEIGHT,WIDTH,-1))
+                
+                rslt = eval_results(result_file, ABSOLUTE_DISPARITY,radius=CLUSTER_RADIUS)                
+                img_gain_test0 = rslt[0][0]/rslt[0][1]   
+                img_gain_test9 = rslt[9][0]/rslt[9][1]   
+     
+     # Close writers
+    train_writer.close()
+    test_writer.close()
+    test_writer1.close()
+#reports error: Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7efc5f720ef0>> if there is no print before exit()
+
+print("All done")
+exit (0)