test_nn_feed.py 8.72 KB
Newer Older
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
#!/usr/bin/env python3

__copyright__ = "Copyright 2018, Elphel, Inc."
__license__   = "GPL-3.0+"
__email__     = "oleg@elphel.com"

'''
Open all tiffs in a folder, combine a single tiff from randomly selected
tiles from originals
'''

from PIL import Image

import os
import sys
import glob

import imagej_tiff as ijt
import pack_tile as pile

import numpy as np
import itertools

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
24 25
import time

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
26 27
import matplotlib.pyplot as plt

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[38;5;214m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    BOLDWHITE = '\033[1;37m'
    UNDERLINE = '\033[4m'


def print_time():
  print(bcolors.BOLDWHITE+"time: "+str(time.time())+bcolors.ENDC)

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
44 45 46
# USAGE: python3 test_3.py some-path

VALUES_LAYER_NAME = 'other'
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
47 48
LAYERS_OF_INTEREST = ['diagm-pair', 'diago-pair', 'hor-pairs', 'vert-pairs']
RADIUS = 1
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
49
TILE_PACKING_TYPE = 1
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
50

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
51
DEBUG_PLT_LOSS = True
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
52 53 54
# If false - will not pack or rescal
DEBUG_PACK_TILES = True

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
55 56 57 58 59
try:
  src = sys.argv[1]
except IndexError:
  src = "."

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
60
print("Importing TensorCrawl")
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
61 62
print_time()

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
63 64
import tensorflow as tf
import tensorflow.contrib.slim as slim
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
65

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
66
print("TensorCrawl imported")
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
67
print_time()
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
68

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
69 70 71 72
tlist = glob.glob(src+"/*.tiff")
print("Found "+str(len(tlist))+" preprocessed tiff files:")
print("\n".join(tlist))
print_time()
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
73

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
74
tiff = ijt.imagej_tiff(tlist[0])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
75

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
76 77 78
# get labels
labels = tiff.labels.copy()
labels.remove(VALUES_LAYER_NAME)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
79

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
80 81 82
print("Image data layers:  "+str(labels))
print("Layers of interest: "+str(LAYERS_OF_INTEREST))
print("Values layer: "+str([VALUES_LAYER_NAME]))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
83 84 85


result_dir = './result/'
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
86
checkpoint_dir = './result/'
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
87 88 89
save_freq = 500

def lrelu(x):
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
90 91
    #return tf.maximum(x*0.2,x)
    return tf.nn.relu(x)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
92 93 94

def network(input):

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
95
  fc1  = slim.fully_connected(input,1024,activation_fn=lrelu,scope='g_fc1')
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
96 97
  fc2  = slim.fully_connected(fc1,     2,activation_fn=lrelu,scope='g_fc2')
  return fc2
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
98

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
99 100 101 102 103
  #fc2  = slim.fully_connected(fc1,  1024,activation_fn=lrelu,scope='g_fc2')
  #fc3  = slim.fully_connected(fc2,   512,activation_fn=lrelu,scope='g_fc3')
  #fc4  = slim.fully_connected(fc3,     8,activation_fn=lrelu,scope='g_fc4')
  #fc5  = slim.fully_connected(fc4,     4,activation_fn=lrelu,scope='g_fc5')
  #fc6  = slim.fully_connected(fc5,     2,activation_fn=lrelu,scope='g_fc6')
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
104

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
105
  #return fc6
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
106 107


Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
108
sess = tf.Session()
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
109

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
110 111 112 113 114 115
if   TILE_PACKING_TYPE==1:
  in_tile = tf.placeholder(tf.float32,[None,101])
elif TILE_PACKING_TYPE==2:
  in_tile = tf.placeholder(tf.float32,[None,105])

gt = tf.placeholder(tf.float32,[None,2])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
116 117 118 119 120 121 122 123 124 125


#losses    = tf.get_variable("losses", [None])
#update_operation = tf.assign(losses,tf.concat([losses,G_loss]))
#mean_loss = tf.reduce_mean(losses)

#tf.summary.scalar('gt_value', gt[0])
#tf.summary.scalar('gt_confidence', gt[1])
#tf.summary.scalar('gt_value',gt[0,0])

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
126
#cf_cutoff = tf.constant(tf.float32,[None,1])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
127 128
out = network(in_tile)

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
129 130 131
#tf.summary.scalar('out_value', out[0,0])
#tf.summary.scalar('out_confidence', out[1])

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
132 133 134
# min cutoff
cf_cutoff = 0.173303
cf_w = tf.pow(tf.maximum(gt[:,1]-cf_cutoff,0.0),1)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
135 136 137
#cf_wsum = tf.reduce_sum(cf_w[~tf.is_nan(cf_w)])
#cf_w_norm = cf_w/cf_wsum
cf_w_norm = tf.nn.softmax(cf_w)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
138 139 140

#out_cf = out[:,1]

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
141
#G_loss = tf.reduce_mean(tf.abs(tf.nn.softmax(out[:,1])*out[:,0]-cf_w_norm*gt[:,0]))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
142
#G_loss = tf.reduce_mean(tf.squared_difference(out[:,0], gt[:,0]))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
143 144
G_loss = tf.reduce_mean(tf.abs(out[:,0]-gt[:,0]))
#G_loss = tf.losses.mean_squared_error(gt[:,0],out[:,0],cf_w)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
145

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
146 147 148
#tf.summary.scalar('loss', G_loss)
#tf.summary.scalar('prediction', out[0,0])
#tf.summary.scalar('ground truth', gt[0,0])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
149

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
150 151
t_vars=tf.trainable_variables()
lr=tf.placeholder(tf.float32)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
152
G_opt=tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
153 154

saver=tf.train.Saver()
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
155 156

# ?!!!!!
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
157 158 159
#merged = tf.summary.merge_all()
#train_writer = tf.summary.FileWriter(result_dir + '/train', sess.graph)
#test_writer = tf.summary.FileWriter(result_dir + '/test')
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
160

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
161 162 163 164 165 166 167 168 169 170 171 172 173
sess.run(tf.global_variables_initializer())
ckpt=tf.train.get_checkpoint_state(checkpoint_dir)

if ckpt:
  print('loaded '+ckpt.model_checkpoint_path)
  saver.restore(sess,ckpt.model_checkpoint_path)


allfolders = glob.glob('./result/*0')
lastepoch = 0
for folder in allfolders:
  lastepoch = np.maximum(lastepoch, int(folder[-4:]))

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
174 175 176 177 178 179 180 181 182
recorded_loss = []
recorded_mean_loss = []

recorded_gt_d = []
recorded_gt_c = []

recorded_pr_d = []
recorded_pr_c = []

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
183
LR = 1e-3
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
184 185 186

print(bcolors.HEADER+"Last Epoch = "+str(lastepoch)+bcolors.ENDC)

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
187 188 189 190 191 192
if DEBUG_PLT_LOSS:
  plt.ion()   # something about plotting
  plt.figure(1, figsize=(4,12))
  pass


Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
193 194
training_tiles  = np.array([])
training_values = np.array([])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
195

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
196 197
# get epoch train data
for i in range(len(tlist)):
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
198

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
199
  print(bcolors.OKGREEN+"Opening "+tlist[i]+bcolors.ENDC)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
200

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
201 202 203 204 205 206 207 208
  tmp_tiff  = ijt.imagej_tiff(tlist[i])
  tmp_tiles = tmp_tiff.getstack(labels,shape_as_tiles=True)
  tmp_vals  = tmp_tiff.getvalues(label=VALUES_LAYER_NAME)
  # might not need it because going to loop through anyway
  if   TILE_PACKING_TYPE==1:
    packed_tiles = pile.pack(tmp_tiles)
  elif TILE_PACKING_TYPE==2:
    packed_tiles = pile.pack(tmp_tiles,TILE_PACKING_TYPE)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
209

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
210
  packed_tiles = np.dstack((packed_tiles,tmp_vals[:,:,0]))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
211

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
212 213
  packed_tiles = np.reshape(packed_tiles,(-1,packed_tiles.shape[-1]))
  values       = np.reshape(tmp_vals[:,:,1:3],(-1,2))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
214

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
215
  packed_tiles_filtered = np.array([])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
216

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
217
  print("Unfiltered: "+str(packed_tiles.shape))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
218

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
219
  for j in range(packed_tiles.shape[0]):
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
220

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
221 222 223 224 225
    skip_tile = False
    if np.isnan(np.sum(packed_tiles[j])):
      skip_tile = True
    if np.isnan(np.sum(values[j])):
      skip_tile = True
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
226

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
227 228 229 230 231 232 233
    if not skip_tile:
      if len(packed_tiles_filtered)==0:
        packed_tiles_filtered = np.array([packed_tiles[j]])
        values_filtered       = np.array([values[j]])
      else:
        packed_tiles_filtered = np.append(packed_tiles_filtered,[packed_tiles[j]],axis=0)
        values_filtered       = np.append(values_filtered,[values[j]],axis=0)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
234

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
235
  print("NaN-filtered: "+str(packed_tiles_filtered.shape))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
236

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
237 238 239 240 241 242
  if i==0:
    training_tiles  = packed_tiles_filtered
    training_values = values_filtered
  else:
    training_tiles  = np.concatenate((training_tiles,packed_tiles_filtered),axis=0)
    training_values = np.concatenate((training_values,values_filtered),axis=0)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
243

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
244
print("Training set shape: "+str(training_tiles.shape))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
245

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
246 247 248
# RUN
# epoch is all available images
# batch is a number of non-zero tiles
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
249

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
250
g_loss = np.zeros(training_tiles.shape[0])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
251

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
252 253
#for epoch in range(lastepoch,lastepoch+len(tlist)):
for epoch in range(lastepoch,500):
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
254

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
255
  print(bcolors.HEADER+"Epoch #"+str(epoch)+bcolors.ENDC)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
256

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
257 258
  if os.path.isdir("result/%04d"%epoch):
    continue
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
259

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
260 261
  #if epoch > 2000:
  #  LR = 1e-5
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
262

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
263 264 265
  # so, here get the image, remove nans and run for 100x times
  #packed_tiles[np.isnan(packed_tiles)] = 0.0
  #tmp_vals[np.isnan(tmp_vals)] = 0.0
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
266

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
267 268
  input_patch = training_tiles[::2]
  gt_patch    = training_values[::2]
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
269

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
270
  st=time.time()
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
271

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
272 273 274
  #run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
  #run_metadata = tf.RunMetadata()
  #_,G_current,output = sess.run([G_opt,G_loss,out],feed_dict={in_tile:input_patch,gt:gt_patch,lr:LR},options=run_options,run_metadata=run_metadata)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
275

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
276
  _,G_current,output = sess.run([G_opt,G_loss,out],feed_dict={in_tile:input_patch,gt:gt_patch,lr:LR})
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
277

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
278 279
  g_loss[i]=G_current
  mean_loss = np.mean(g_loss[np.where(g_loss)])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
280

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
281
  if DEBUG_PLT_LOSS:
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
282

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
283 284
    recorded_loss.append(G_current)
    recorded_mean_loss.append(mean_loss)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
285

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
286 287
    recorded_pr_d.append(output[0,0])
    recorded_pr_c.append(output[0,1])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
288

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
289 290
    recorded_gt_d.append(gt_patch[0,0])
    recorded_gt_c.append(gt_patch[0,1])
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
291

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
292
    plt.clf()
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
293

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
294
    plt.subplot(311)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
295

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
296 297 298 299 300 301
    plt.plot(recorded_loss,  label='loss')
    plt.plot(recorded_mean_loss,  label='mean loss', color='red')
    plt.xlabel('Iteration')
    plt.ylabel('Loss')
    plt.title("Loss=%.5f, Mean Loss=%.5f"%(G_current,mean_loss), fontdict={'size': 20, 'color': 'red'})
    #plt.text(0.5, 0.5, 'Loss=%.5f' % G_current, fontdict={'size': 20, 'color': 'red'})
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
302

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
303
    plt.subplot(312)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
304

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
305 306 307 308 309
    plt.xlabel('Iteration')
    plt.ylabel('Disparities')
    plt.plot(recorded_gt_d,  label='gt_d',color='green')
    plt.plot(recorded_pr_d,  label='pr_d',color='red')
    plt.legend(loc='best',ncol=1)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
310

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
311
    plt.subplot(313)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
312

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
313 314 315 316 317
    plt.xlabel('Iteration')
    plt.ylabel('Confidences')
    plt.plot(recorded_gt_c,  label='gt_c',color='green')
    plt.plot(recorded_pr_c,  label='pr_c',color='red')
    plt.legend(loc='best',ncol=1)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
318

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
319
    plt.pause(0.001)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
320

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
321 322
  else:
    print("%d %d Loss=%.3f CurrentLoss=%.3f Time=%.3f"%(epoch,i,mean_loss,G_current,time.time()-st))
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
323

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
324

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
325 326 327
  if epoch%save_freq==0:
    if not os.path.isdir(result_dir + '%04d'%epoch):
      os.makedirs(result_dir + '%04d'%epoch)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
328 329 330 331 332

  saver.save(sess, checkpoint_dir + 'model.ckpt')

print_time()
print(bcolors.OKGREEN+"time: "+str(time.time())+bcolors.ENDC)
Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
333

Oleg Dzhimiev's avatar
Oleg Dzhimiev committed
334 335
plt.ioff()
plt.show()