Added pydev project setup, calculation of the source data stats for

batch equalization

.gitignore

 __pycache__
+/.project
+/.pydevproject
 attic
 *.log
\ No newline at end of file

eclipse_project_setup/.project

+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>python3-imagej-tiff</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

eclipse_project_setup/.pydevproject

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
+</pydev_project>

imagej_tiff.py

@@ -102,7 +102,7 @@ class imagej_tiff:
   __TIFF_TAG_LABELS_STRINGS = 50839
 
   # init
-  def __init__(self,filename):
+  def __init__(self,filename, layers = None):
     # file name
     self.fname = filename
     tif = Image.open(filename)
@@ -123,32 +123,43 @@ class imagej_tiff:
     # image layers stacked along depth - (think RGB)
     self.image = []
 
+    if layers is None:
     # fill self.image
-    for i in range(self.nimages):
-      tif.seek(i)
-      a = np.array(tif)
-      a = np.reshape(a,(a.shape[0],a.shape[1],1))
-
-      #a = a[:,:,np.newaxis]
-
-      # scale for 8-bits
-      # exclude layer named 'other'
-      if self.bpp==8:
-        _min = self.data_min
-        _max = self.data_max
-        _MIN = 1
-        _MAX = 255
-        a = a.astype(float)
-        if self.labels[i]!='other':
-          a[a==0]=np.nan
-          a = (_max-_min)*(a-_MIN)/(_MAX-_MIN)+_min
-
-      # init
-      if i==0:
-        self.image = a
-      # stack along depth (think of RGB channels)
-      else:
-        self.image = np.append(self.image,a,axis=2)
+        for i in range(self.nimages):
+          tif.seek(i)
+          a = np.array(tif)
+          a = np.reshape(a,(a.shape[0],a.shape[1],1))
+    
+          #a = a[:,:,np.newaxis]
+    
+          # scale for 8-bits
+          # exclude layer named 'other'
+          if self.bpp==8:
+            _min = self.data_min
+            _max = self.data_max
+            _MIN = 1
+            _MAX = 255
+            a = a.astype(float)
+            if self.labels[i]!='other':
+              a[a==0]=np.nan
+              a = (_max-_min)*(a-_MIN)/(_MAX-_MIN)+_min
+    
+          # init
+          if i==0:
+            self.image = a
+          # stack along depth (think of RGB channels)
+          else:
+            self.image = np.append(self.image,a,axis=2)
+    else:
+        indx = 0
+        for layer in layers:
+            tif.seek(self.labels.index(layer)) 
+            a = np.array(tif)
+            if not indx:
+                self.image = np.empty((a.shape[0],a.shape[1],len(layers)),a.dtype)
+            self.image[...,indx] = a
+            indx += 1
+        
 
     # init done, close the image
     tif.close()
@@ -266,17 +277,18 @@ class imagej_tiff:
     # properties dictionary
     pd = {}
 
-    for child in infos[0]:
-      #print(child.tag+"::::::"+child.text)
-      pd[child.tag] = child.text
-
-    self.props = pd
-
-    # tiles are squares
-    self.tileW = int(self.props['tileWidth'])
-    self.tileH = int(self.props['tileWidth'])
-    self.data_min = float(self.props['data_min'])
-    self.data_max = float(self.props['data_max'])
+    if infos:
+        for child in infos[0]:
+          #print(child.tag+"::::::"+child.text)
+          pd[child.tag] = child.text
+    
+        self.props = pd
+    
+        # tiles are squares
+        self.tileW = int(self.props['tileWidth'])
+        self.tileH = int(self.props['tileWidth'])
+        self.data_min = float(self.props['data_min'])
+        self.data_max = float(self.props['data_max'])
 
   # makes arrays of labels (strings) and unparsed xml infos
   def __split_labels(self,n,tag):

test_explore_data.py

+#!/usr/bin/env python3
+from numpy import float64
+
+__copyright__ = "Copyright 2018, Elphel, Inc."
+__license__   = "GPL-3.0+"
+__email__     = "andrey@elphel.com"
+
+import os
+import sys
+import glob
+import imagej_tiff as ijt
+import numpy as np
+import resource
+import timeit
+import matplotlib.pyplot as plt
+from scipy.ndimage.filters import gaussian_filter
+
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+def print_time():
+  print(bcolors.BOLDWHITE+"time: "+str(time.time())+bcolors.ENDC)
+
+def getComboList(top_dir):
+    patt = "*-DSI_COMBO.tiff"
+    tlist = []
+    for i in range(5):
+        pp = top_dir#) ,'**', patt) # works
+        for j in range (i):
+            pp = os.path.join(pp,'*')
+        pp = os.path.join(pp,patt)
+        tlist += glob.glob(pp)
+        print (pp+" "+str(len(tlist)))
+    print("Found "+str(len(tlist))+" preprocessed tiff files:")
+    print("\n".join(tlist))
+    return tlist
+def loadComboFiles(tlist):
+    indx = 0
+    images = []
+    print(str(resource.getrusage(resource.RUSAGE_SELF)))
+    for combo_file in tlist:
+        tiff = ijt.imagej_tiff(combo_file,['disparity_rig','strength_rig'])
+        if not indx:
+            images = np.empty((len(tlist), tiff.image.shape[0],tiff.image.shape[1],tiff.image.shape[2]), tiff.image.dtype)
+        images[indx] = tiff.image
+        print(str(indx)+": "+str(resource.getrusage(resource.RUSAGE_SELF)))
+        indx += 1
+    return images
+
+def getHistogramDSI(
+        combo_rds,
+        disparity_bins =    1000,
+        strength_bins =      100,
+        disparity_min_drop =  -0.1,
+        disparity_min_clip =  -0.1,
+        disparity_max_drop = 100.0,
+        disparity_max_clip = 100.0,
+        strength_min_drop =    0.1,
+        strength_min_clip =    0.1,
+        strength_max_drop =    1.0,
+        strength_max_clip =    0.9,
+        normalize =           True
+        ):
+    good_tiles = np.empty((combo_rds.shape[0], combo_rds.shape[1],combo_rds.shape[2]), dtype=bool)
+    for ids in range (combo_rds.shape[0]): #iterate over all scenes ds[2][rows][cols]
+        ds = combo_rds[ids]
+        disparity = ds[...,0]
+        strength =  ds[...,1]
+        good_tiles[ids] =  disparity >= disparity_min_drop
+        good_tiles[ids] &= disparity <= disparity_max_drop
+        good_tiles[ids] &= strength >=  strength_min_drop
+        good_tiles[ids] &= strength <=  strength_max_drop
+        disparity = np.nan_to_num(disparity, copy = False) # to be able to multiply by 0.0 in mask | copy=False, then out=disparity all done in-place
+        strength =  np.nan_to_num(strength, copy = False)  # likely should never happen
+        np.clip(disparity, disparity_min_clip, disparity_max_clip, out = disparity)
+        np.clip(strength, strength_min_clip, strength_max_clip, out = strength)
+        pass
+    hist, xedges, yedges = np.histogram2d( # xedges, yedges - just for debugging
+        x = combo_rds[...,0].flatten(),
+        y = combo_rds[...,1].flatten(),
+        bins=(disparity_bins, strength_bins),
+        range=((disparity_min_clip,disparity_max_clip),(strength_min_clip,strength_max_clip)),
+        normed=normalize,
+        weights=good_tiles.flatten())
+    return hist, xedges, yedges
+
+#MAIN
+if __name__ == "__main__":
+  try:
+    top_dir = sys.argv[1]
+  except IndexError:
+    top_dir = "/mnt/dde6f983-d149-435e-b4a2-88749245cc6c/home/eyesis/x3d_data/data_sets/all"#test" #all/"
+  tlist = getComboList(top_dir)
+  pre_log_offs =        0.001
+  combo_rds= loadComboFiles(tlist)
+  hist, xedges, yedges = getHistogramDSI(combo_rds)
+  lhist = np.log(hist.transpose() + pre_log_offs)
+  
+
+  blurred = gaussian_filter(lhist, sigma=2.0)
+  
+  
+  mytitle = "Disparity/Strength histogram"
+  fig = plt.figure()
+  fig.canvas.set_window_title("Window title: "+mytitle)
+  fig.suptitle("Window subtitle: "+mytitle)
+#  plt.imshow(lhist,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  plt.imshow(blurred)#,vmin=-6,vmax=-2) # , vmin=0, vmax=.01)
+  plt.colorbar()
+  plt.ioff()
+  plt.show()
+  pass
+