Commit 5591b717 by Oleg Dzhimiev

### preparing to feed nn

parent c21e72b7
 ... ... @@ -67,3 +67,41 @@ def pack_tile(tile,lut): out = np.append(out,layer) return out # tiles are already packed def get_tile_with_neighbors(tiles,i,j,radius): out = np.array([]) # max Y,X = tiles.shape[0:2] #print(str(Y)+" "+str(X)) for k in range(2*radius+1): y = i+k-radius if y<0: y = 0 elif y>(Y-1): y = Y-1 for l in range(2*radius+1): x = j+l-radius if x<0: x = 0 elif x>(X-1): x = X-1 out = np.append(out,tiles[y,x]) return out
 ... ... @@ -8,6 +8,8 @@ __email__ = "oleg@elphel.com" Open all tiffs in a folder, combine a single tiff from randomly selected tiles from originals ''' import tensorflow as tf #import tensorflow.contrib.slim as slim from PIL import Image ... ... @@ -21,32 +23,49 @@ import pack_tile as pile import numpy as np import itertools import time sys.exit() #http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' OKGREEN = '\033[92m' WARNING = '\033[38;5;214m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' BOLDWHITE = '\033[1;37m' UNDERLINE = '\033[4m' def print_time(): print(bcolors.BOLDWHITE+"time: "+str(time.time())+bcolors.ENDC) # USAGE: python3 test_3.py some-path VALUES_LAYER_NAME = 'other' LAYERS_OF_INTEREST = ['diagm-pair', 'diago-pair', 'hor-pairs', 'vert-pairs'] RADIUS = 1 try: src = sys.argv[1] except IndexError: src = "." print_time() tlist = glob.glob(src+"/*.tiff") print("Found "+str(len(tlist))+" tiff files:") print("Found "+str(len(tlist))+" preprocessed tiff files:") print("\n".join(tlist)) print_time() ''' WARNING, assuming: - timestamps and part of names match - layer order and names are identical ''' # CONSTANTS RADIUS = 1 LAYERS_OF_INTEREST = ['diagm-pair','diago-pair'] # open the first one to get dimensions and other info tiff = ijt.imagej_tiff(tlist[0]) #del tlist[0] ... ... @@ -58,15 +77,18 @@ tiff = ijt.imagej_tiff(tlist[0]) labels = tiff.labels.copy() labels.remove(VALUES_LAYER_NAME) print("Image data layers: "+str(labels)) print("Image data layers: "+str(labels)) print("Layers of interest: "+str(LAYERS_OF_INTEREST)) print("Values layer: "+str([VALUES_LAYER_NAME])) # create copies tiles = np.copy(tiff.getstack(labels,shape_as_tiles=True)) tiles_bkp = np.copy(tiles) values = np.copy(tiff.getvalues(label=VALUES_LAYER_NAME)) print("Tiled tiff shape: "+str(tiles.shape)) #gt = values[:,:,1:3] print("Mixed tiled input data shape: "+str(tiles.shape)) #print_time() # now generate a layer of indices to get other tiles indices = np.random.random_integers(0,len(tlist)-1,size=(tiles.shape[0],tiles.shape[1])) ... ... @@ -97,7 +119,8 @@ for i in range(1,len(tlist)): for i in range(1,len(shuffle_counter)): shuffle_counter[0] -= shuffle_counter[i] print("Tiffs shuffle counter = "+str(shuffle_counter)) print("Tiff files parts count in the mixed input = "+str(shuffle_counter)) print_time() # test later ... ... @@ -107,7 +130,77 @@ print("Tiffs shuffle counter = "+str(shuffle_counter)) # Parse packing table # packing table name ptab_name = "tile_packing_table.xml" pt = pile.PackingTable(ptab_name,LAYERS_OF_INTEREST).lut ptab = pile.PackingTable(ptab_name,LAYERS_OF_INTEREST).lut # might not need it because going to loop through anyway packed_tiles = np.array([[pile.pack_tile(tiles[i,j],ptab) for j in range(tiles.shape[1])] for i in range(tiles.shape[0])]) packed_tiles = np.dstack((packed_tiles,values[:,:,0])) print("Packed (81x4 -> 1x(25*4+1)) tiled input shape: "+str(packed_tiles.shape)) print_time() #for i in range(tiles.shape[0]): # for j in range(tiles.shape[1]): # nn_input = pile.get_tile_with_neighbors(tiles,i,j,RADIUS) # print("tile: "+str(i)+", "+str(j)+": shape = "+str(nn_input.shape)) #print_time() result_dir = './result/' save_freq = 500 def lrelu(x): return tf.maximum(x*0.2,x) def network(input): fc1 = slim.fully_connected(input,42,activation_fn=lrelu,scope='g_fc1') fc2 = slim.fully_connected(fc1, 21,activation_fn=lrelu,scope='g_fc2') fc3 = slim.fully_connected(fc2, 1,activation_fn=lrelu,scope='g_fc3') return fc3 sess = tf.session() in_tile = tf.placeholder(tf.float32,[None,None,101]) gt = tf.placeholder(tf.float32,[None,None,2]) out = network(in_tile) G_loss = tf.reduce_mean(tf.abs(out[:,0]-gt[:,0])) t_vars=tf.trainable_variables() lr=tf.placeholder(tf.float32) G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss,var_list=[var for var in t_vars if var.name.startswith('g_')]) saver=tf.train.Saver() sess.run(tf.global_variables_initializer()) ckpt=tf.train.get_checkpoint_state(checkpoint_dir) if ckpt: print('loaded '+ckpt.model_checkpoint_path) saver.restore(sess,ckpt.model_checkpoint_path) allfolders = glob.glob('./result/*0') lastepoch = 0 for folder in allfolders: lastepoch = np.maximum(lastepoch, int(folder[-4:])) #g_loss = np.zeros((,1)) learning_rate = 1e-4 for epoch in range(lastepoch,4001): if os.path.isdir("result/%04d"%epoch): continue cnt=0 if epoch > 2000: learning_rate = 1e-5 for ind in np.random.permutation(tiles.shape[0]*tiles.shape[1]): input_patch = tiles[i,j] gt_patch = values[i,j,1:2] ... ...
test_tf_setup.py 0 → 100644
 #!/usr/bin/env python3 __copyright__ = "Copyright 2018, Elphel, Inc." __license__ = "GPL-3.0+" __email__ = "oleg@elphel.com" ''' Test: nvidia graphic card cuda installation tensorflow Comment: With nvidia + tensorflow - any software update casually breaks everything ''' import subprocess import re #http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' OKGREEN = '\033[92m' WARNING = '\033[38;5;214m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' BOLDWHITE = '\033[1;37m' UNDERLINE = '\033[4m' # STEP 1: print nvidia model print(bcolors.BOLDWHITE+"NVIDIA devices:"+bcolors.ENDC) p = subprocess.run("lspci | grep NVIDIA",shell=True,stdout=subprocess.PIPE) out = p.stdout.strip().decode() if len(out)==0: print(bcolors.FAIL+" not found (try 'lspci')"+bcolors.ENDC) else: print(out) # STEP 2: nvidia driver version print(bcolors.BOLDWHITE+"NVIDIA driver version:"+bcolors.ENDC) p = subprocess.run("cat /proc/driver/nvidia/version",shell=True,stdout=subprocess.PIPE) out = p.stdout.strip().decode() print(out) # STEP 3: nvidia-smi - also some information about the graphics card and the driver print(bcolors.BOLDWHITE+"Some more info from 'nvidia-smi':"+bcolors.ENDC) p = subprocess.run("nvidia-smi",shell=True,stdout=subprocess.PIPE) out = p.stdout.strip().decode() print(out) print(bcolors.OKGREEN+"END"+bcolors.ENDC) \ No newline at end of file
 ... ... @@ -53,4 +53,58 @@
[(66,1.0), (67,1.0), (68,1.0)][(74,1.0), (75,1.0), (76,1.0), (77,1.0), (78,1.0)]
[(2,1.0), (3,1.0), (4,1.0), (5,1.0), (6,1.0)][(10,1.0), (11,1.0), (19,1.0), (20,1.0)][(12,1.0), (13,1.0), (14,1.0)][(15,1.0), (16,1.0), (24,1.0), (25, 1.0)][(18,1.0), (27,1.0), (36,1.0), (45,1.0), (54,1.0)][(21,1.0), (22,1.0), (23,1.0)][(26,1.0), (35,1.0), (44,1.0), (53,1.0), (62,1.0)][(28,1.0), (37,1.0), (46,1.0)][(29,1.0), (38,1.0), (47,1.0)][(30,1.0)][(31,1.0)][(32,1.0)][(33,1.0), (42,1.0), (51,1.0)][(34,1.0), (43,1.0), (52,1.0)][(39,1.0)][(40,1.0)][(41,1.0)][(48,1.0)][(49,1.0)][(50,1.0)][(55,1.0), (56,1.0), (64,1.0), (65,1.0)][(57,1.0), (58,1.0), (59,1.0)][(60,1.0), (61,1.0), (69,1.0), (70,1.0)][(66,1.0), (67,1.0), (68,1.0)][(74,1.0), (75,1.0), (76,1.0), (77,1.0), (78,1.0)]
[(2,1.0), (3,1.0), (4,1.0), (5,1.0), (6,1.0)][(10,1.0), (11,1.0), (19,1.0), (20,1.0)][(12,1.0), (13,1.0), (14,1.0)][(15,1.0), (16,1.0), (24,1.0), (25, 1.0)][(18,1.0), (27,1.0), (36,1.0), (45,1.0), (54,1.0)][(21,1.0), (22,1.0), (23,1.0)][(26,1.0), (35,1.0), (44,1.0), (53,1.0), (62,1.0)][(28,1.0), (37,1.0), (46,1.0)][(29,1.0), (38,1.0), (47,1.0)][(30,1.0)][(31,1.0)][(32,1.0)][(33,1.0), (42,1.0), (51,1.0)][(34,1.0), (43,1.0), (52,1.0)][(39,1.0)][(40,1.0)][(41,1.0)][(48,1.0)][(49,1.0)][(50,1.0)][(55,1.0), (56,1.0), (64,1.0), (65,1.0)][(57,1.0), (58,1.0), (59,1.0)][(60,1.0), (61,1.0), (69,1.0), (70,1.0)][(66,1.0), (67,1.0), (68,1.0)][(74,1.0), (75,1.0), (76,1.0), (77,1.0), (78,1.0)] \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!