Commit cd541734 authored by Oleg Dzhimiev's avatar Oleg Dzhimiev
Browse files

simplified packing, next test

parent 51325319

imagej_tiff.py

+1 −1
Original line number Original line Diff line number Diff line
@@ -399,7 +399,7 @@ if __name__ == "__main__": 
  #ijt.show_images(['X-corr','Y-corr'])

  #ijt.show_images(['X-corr','Y-corr'])

  #ijt.show_images(['R-vign',3])

  #ijt.show_images(['R-vign',3])





  ijt.show_images([0])

  ijt.show_images()

  plt.show()

  plt.show()

pack_tile.py

+121 −83
Original line number Original line Diff line number Diff line
@@ -5,86 +5,134 @@ __license__ = "GPL-3.0+" 
__email__     = "oleg@elphel.com"

__email__     = "oleg@elphel.com"





import numpy as np

import numpy as np

import xml.etree.ElementTree as ET

import ast





class PackingTable:

# pack from 9x9x4 to 25x1

def pack_tile_type1(tile):





  def __init__(self,filename,layers_of_interest):

  out = np.empty(100)

    e = ET.parse(filename).getroot()

    #print(ET.tostring(e))

    #reparsed = minidom.parseString(ET.tostring(e,""))

    #print(reparsed.toprettyxml(indent="\t"))



    # Parse xml:

    # td = tmp_dict

    td = {}

    for table in e:

      layer = table.get('layer')

      td[layer] = []

      for row in table:

        # safe evaluation

        td[layer].append(ast.literal_eval(row.text))



    # order

    LUT = []

    for layer in layers_of_interest:

      LUT.append(td[layer])



    self.lut = LUT





# A tile consists of layers

# layer is packed from 9x9 to 25x1

def pack_layer(layer,lut_row):



  #print(layer.shape)



  t = layer.flatten()



  #out = np.array([])

  out = np.empty(len(lut_row))



  # iterate through rows

  for i in range(len(lut_row)):



    val = 0

    # process row value

    for j in lut_row[i]:

      ## big slowdown

      #if np.isnan(t[j[0]]):

      # val = np.nan

      #  break

      #val += t[j[0]]*j[1]

      val += t[j[0]]





    #out = np.append(out,val)

  # pack diagm-pair

    #out[i] = val

  l = np.ravel(tile[:,:,0])

    out[i] = val

  out[ 0] = 1.0*l[ 2]+1.0*l[ 3]+1.0*l[ 4]+1.0*l[ 5]+1.0*l[6]

  out[ 1] = 1.0*l[10]+1.0*l[11]+1.0*l[19]+1.0*l[20]

  out[ 2] = 1.0*l[12]+1.0*l[13]+1.0*l[14]

  out[ 3] = 1.0*l[15]+1.0*l[16]+1.0*l[24]+1.0*l[25]

  out[ 4] = 1.0*l[18]+1.0*l[27]+1.0*l[36]+1.0*l[45]+1.0*l[54]

  out[ 5] = 1.0*l[21]+1.0*l[22]+1.0*l[23]

  out[ 6] = 1.0*l[26]+1.0*l[35]+1.0*l[44]+1.0*l[53]+1.0*l[62]

  out[ 7] = 1.0*l[28]+1.0*l[37]+1.0*l[46]

  out[ 8] = 1.0*l[29]+1.0*l[38]+1.0*l[47]

  out[ 9] = 1.0*l[30]

  out[10] = 1.0*l[31]

  out[11] = 1.0*l[32]

  out[12] = 1.0*l[33]+1.0*l[42]+1.0*l[51]

  out[13] = 1.0*l[34]+1.0*l[43]+1.0*l[52]

  out[14] = 1.0*l[39]

  out[15] = 1.0*l[40]

  out[16] = 1.0*l[41]

  out[17] = 1.0*l[48]

  out[18] = 1.0*l[49]

  out[19] = 1.0*l[50]

  out[20] = 1.0*l[55]+1.0*l[56]+1.0*l[64]+1.0*l[65]

  out[21] = 1.0*l[57]+1.0*l[58]+1.0*l[59]

  out[22] = 1.0*l[60]+1.0*l[61]+1.0*l[69]+1.0*l[70]

  out[23] = 1.0*l[66]+1.0*l[67]+1.0*l[68]

  out[24] = 1.0*l[74]+1.0*l[75]+1.0*l[76]+1.0*l[77]+1.0*l[78]



  # pack diago-pair

  l = np.ravel(tile[:,:,1])

  out[25] = 1.0*l[ 2]+1.0*l[ 3]+1.0*l[ 4]+1.0*l[ 5]+1.0*l[6]

  out[26] = 1.0*l[10]+1.0*l[11]+1.0*l[19]+1.0*l[20]

  out[27] = 1.0*l[12]+1.0*l[13]+1.0*l[14]

  out[28] = 1.0*l[15]+1.0*l[16]+1.0*l[24]+1.0*l[25]

  out[29] = 1.0*l[18]+1.0*l[27]+1.0*l[36]+1.0*l[45]+1.0*l[54]

  out[30] = 1.0*l[21]+1.0*l[22]+1.0*l[23]

  out[31] = 1.0*l[26]+1.0*l[35]+1.0*l[44]+1.0*l[53]+1.0*l[62]

  out[32] = 1.0*l[28]+1.0*l[37]+1.0*l[46]

  out[33] = 1.0*l[29]+1.0*l[38]+1.0*l[47]

  out[34] = 1.0*l[30]

  out[35] = 1.0*l[31]

  out[36] = 1.0*l[32]

  out[37] = 1.0*l[33]+1.0*l[42]+1.0*l[51]

  out[38] = 1.0*l[34]+1.0*l[43]+1.0*l[52]

  out[39] = 1.0*l[39]

  out[40] = 1.0*l[40]

  out[41] = 1.0*l[41]

  out[42] = 1.0*l[48]

  out[43] = 1.0*l[49]

  out[44] = 1.0*l[50]

  out[45] = 1.0*l[55]+1.0*l[56]+1.0*l[64]+1.0*l[65]

  out[46] = 1.0*l[57]+1.0*l[58]+1.0*l[59]

  out[47] = 1.0*l[60]+1.0*l[61]+1.0*l[69]+1.0*l[70]

  out[48] = 1.0*l[66]+1.0*l[67]+1.0*l[68]

  out[49] = 1.0*l[74]+1.0*l[75]+1.0*l[76]+1.0*l[77]+1.0*l[78]



  # pack hor-pairs

  l = np.ravel(tile[:,:,2])

  out[50] = 1.0*l[ 2]+1.0*l[ 3]+1.0*l[ 4]+1.0*l[ 5]+1.0*l[6]

  out[51] = 1.0*l[10]+1.0*l[11]+1.0*l[19]+1.0*l[20]

  out[52] = 1.0*l[12]+1.0*l[13]+1.0*l[14]

  out[53] = 1.0*l[15]+1.0*l[16]+1.0*l[24]+1.0*l[25]

  out[54] = 1.0*l[18]+1.0*l[27]+1.0*l[36]+1.0*l[45]+1.0*l[54]

  out[55] = 1.0*l[21]+1.0*l[22]+1.0*l[23]

  out[56] = 1.0*l[26]+1.0*l[35]+1.0*l[44]+1.0*l[53]+1.0*l[62]

  out[57] = 1.0*l[28]+1.0*l[37]+1.0*l[46]

  out[58] = 1.0*l[29]+1.0*l[38]+1.0*l[47]

  out[59] = 1.0*l[30]

  out[60] = 1.0*l[31]

  out[61] = 1.0*l[32]

  out[62] = 1.0*l[33]+1.0*l[42]+1.0*l[51]

  out[63] = 1.0*l[34]+1.0*l[43]+1.0*l[52]

  out[64] = 1.0*l[39]

  out[65] = 1.0*l[40]

  out[66] = 1.0*l[41]

  out[67] = 1.0*l[48]

  out[68] = 1.0*l[49]

  out[69] = 1.0*l[50]

  out[70] = 1.0*l[55]+1.0*l[56]+1.0*l[64]+1.0*l[65]

  out[71] = 1.0*l[57]+1.0*l[58]+1.0*l[59]

  out[72] = 1.0*l[60]+1.0*l[61]+1.0*l[69]+1.0*l[70]

  out[73] = 1.0*l[66]+1.0*l[67]+1.0*l[68]

  out[74] = 1.0*l[74]+1.0*l[75]+1.0*l[76]+1.0*l[77]+1.0*l[78]



  # pack vert-pairs

  l = np.ravel(tile[:,:,3])

  out[75] = 1.0*l[ 2]+1.0*l[ 3]+1.0*l[ 4]+1.0*l[ 5]+1.0*l[6]

  out[76] = 1.0*l[10]+1.0*l[11]+1.0*l[19]+1.0*l[20]

  out[77] = 1.0*l[12]+1.0*l[13]+1.0*l[14]

  out[78] = 1.0*l[15]+1.0*l[16]+1.0*l[24]+1.0*l[25]

  out[79] = 1.0*l[18]+1.0*l[27]+1.0*l[36]+1.0*l[45]+1.0*l[54]

  out[80] = 1.0*l[21]+1.0*l[22]+1.0*l[23]

  out[81] = 1.0*l[26]+1.0*l[35]+1.0*l[44]+1.0*l[53]+1.0*l[62]

  out[82] = 1.0*l[28]+1.0*l[37]+1.0*l[46]

  out[83] = 1.0*l[29]+1.0*l[38]+1.0*l[47]

  out[84] = 1.0*l[30]

  out[85] = 1.0*l[31]

  out[86] = 1.0*l[32]

  out[87] = 1.0*l[33]+1.0*l[42]+1.0*l[51]

  out[88] = 1.0*l[34]+1.0*l[43]+1.0*l[52]

  out[89] = 1.0*l[39]

  out[90] = 1.0*l[40]

  out[91] = 1.0*l[41]

  out[92] = 1.0*l[48]

  out[93] = 1.0*l[49]

  out[94] = 1.0*l[50]

  out[95] = 1.0*l[55]+1.0*l[56]+1.0*l[64]+1.0*l[65]

  out[96] = 1.0*l[57]+1.0*l[58]+1.0*l[59]

  out[97] = 1.0*l[60]+1.0*l[61]+1.0*l[69]+1.0*l[70]

  out[98] = 1.0*l[66]+1.0*l[67]+1.0*l[68]

  out[99] = 1.0*l[74]+1.0*l[75]+1.0*l[76]+1.0*l[77]+1.0*l[78]





  return out

  return out





# pack single

def pack_tile(tile):

  return pack_tile_type1(tile)





# tile and lut already ordered and indices match

# pack all tiles

def pack_tile(tile,lut):

def pack(tiles):



  output = np.array([[pack_tile(tiles[i,j]) for j in range(tiles.shape[1])] for i in range(tiles.shape[0])])

  #out = np.array([])

  return output

  #s = 0

  #for i in range(len(lut)):

  #  s += len(lut[i])



  #out = np.empty(s)

  out = np.empty(100)



  ptr = 0

  for i in range(len(lut)):

    layer = pack_layer(tile[:,:,i],lut[i])

    #out = np.append(out,layer)

    out[ptr:ptr+len(lut[i])] = layer

    #out[ptr:ptr+len(lut[i])] = np.empty(25)

    ptr += len(lut[i])



  return out









# tiles are already packed

# tiles are already packed
@@ -114,13 +162,3 @@ def get_tile_with_neighbors(tiles,i,j,radius): 
      out = np.append(out,tiles[y,x])

      out = np.append(out,tiles[y,x])





  return out
 
  return out

test_nn_feed.py

+2 −60
Original line number Original line Diff line number Diff line
@@ -141,67 +141,14 @@ if not IS_TEST: 




  # test later

  # test later





  # now pack from 9x9 to 1x25

  # tiles and values



  # Parse packing table

  # packing table name

  ptab_name = "tile_packing_table.xml"

  ptab = pile.PackingTable(ptab_name,LAYERS_OF_INTEREST).lut



  # might not need it because going to loop through anyway

  # might not need it because going to loop through anyway

  packed_tiles = np.array([[pile.pack_tile(tiles[i,j],ptab) for j in range(tiles.shape[1])] for i in range(tiles.shape[0])])

  packed_tiles = pile.pack(tiles)

  packed_tiles = np.dstack((packed_tiles,values[:,:,0]))

  packed_tiles = np.dstack((packed_tiles,values[:,:,0]))





  print("Packed (81x4 -> 1x(25*4+1)) tiled input shape: "+str(packed_tiles.shape))

  print("Packed (81x4 -> 1x(25*4+1)) tiled input shape: "+str(packed_tiles.shape))

  print("Values shape "+str(values.shape))

  print("Values shape "+str(values.shape))

  print_time()

  print_time()





else:

  print("Init test data")



  ptab_name = "tile_packing_table.xml"

  pt = pile.PackingTable(ptab_name,LAYERS_OF_INTEREST).lut



  # 9x9 2 layers, no neighbors

  l = np.zeros((9,9))

  for y,x in itertools.product(range(l.shape[0]),range(l.shape[1])):

    l[y,x] = 9*y + x



  l_value = np.array([2.54,3.54,0.5])



  #print(l)



  l1 = l

  l2 = l*2

  l3 = l*3

  l4 = l*4



  ls = np.dstack((l1,l2,l3,l4))

  #print(ls.shape)



  l_packed_pre = pile.pack_tile(ls,pt)

  #print(l_packed_pre.shape)

  #print(l_packed_pre)



  l_packed = np.hstack((l_packed_pre,l_value[0]))



  #print(l_packed.shape)

  #print(l_packed)

  # use l_packed



  packed_tiles = np.empty([1,1,l_packed.shape[0]])

  values =       np.empty([1,1,2])

  print(packed_tiles.shape)

  print(values.shape)



  packed_tiles[0,0] = l_packed

  values[0,0] = l_value[1:3]



  print(packed_tiles[0,0])

  print(values[0,0])





# END IF IS_TEST

# END IF IS_TEST
@@ -341,13 +288,8 @@ for epoch in range(lastepoch,lastepoch+len(tlist)): 
  tmp_tiles = tmp_tiff.getstack(labels,shape_as_tiles=True)

  tmp_tiles = tmp_tiff.getstack(labels,shape_as_tiles=True)

  tmp_vals  = tmp_tiff.getvalues(label=VALUES_LAYER_NAME)

  tmp_vals  = tmp_tiff.getvalues(label=VALUES_LAYER_NAME)





  # Parse packing table

  # packing table name

  ptab_name = "tile_packing_table.xml"

  ptab = pile.PackingTable(ptab_name,LAYERS_OF_INTEREST).lut



  # might not need it because going to loop through anyway

  # might not need it because going to loop through anyway

  packed_tiles = np.array([[pile.pack_tile(tmp_tiles[i,j],ptab) for j in range(tmp_tiles.shape[1])] for i in range(tmp_tiles.shape[0])])

  packed_tiles = pile.pack(tmp_tiles)

  packed_tiles = np.dstack((packed_tiles,tmp_vals[:,:,0]))

  packed_tiles = np.dstack((packed_tiles,tmp_vals[:,:,0]))





  #if epoch > 2000:

  #if epoch > 2000:

test_nn_infer.py

+1 −6
Original line number Original line Diff line number Diff line
@@ -139,13 +139,8 @@ for item in tlist: 
  # now pack from 9x9 to 1x25

  # now pack from 9x9 to 1x25

  # tiles and values

  # tiles and values





  # Parse packing table

  # packing table name

  ptab_name = "tile_packing_table.xml"

  ptab = pile.PackingTable(ptab_name,LAYERS_OF_INTEREST).lut



  # might not need it because going to loop through anyway

  # might not need it because going to loop through anyway

  packed_tiles = np.array([[pile.pack_tile(tiles[i,j],ptab) for j in range(tiles.shape[1])] for i in range(tiles.shape[0])])

  packed_tiles = pile.pack(tiles)

  packed_tiles = np.dstack((packed_tiles,values[:,:,0]))

  packed_tiles = np.dstack((packed_tiles,values[:,:,0]))





  print(packed_tiles.shape)

  print(packed_tiles.shape)