implemented transfer over ssh (10x slower), restored input(), forced python3

int_ssd_download.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Download footage from ssd's raw partition:
 # * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off
@@ -72,6 +72,7 @@ parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [b
 parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")
 parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")
 parser.add_argument("dest",help="desitnation directory: /data/footage/test")
+parser.add_argument("-l","--lan", action='store_true', help="Transfer data over LAN instead of eSATA")
 
 args = parser.parse_args()
 print (args)
@@ -106,47 +107,69 @@ if len(cams)==0:
 
 pc = x393.PC()
 
-# ssd to camera for all
-for cam in cams:
-  cam['obj'].ssd_to_camera()
 
-# get raw partitions
+
+if (args.lan):
+  print ("Using data transfer over LAN")
+else:  
+  print ("Using data transfer over eSATA, multiplexing camera SSD to eSATA external port")
+# ssd to camera for all
+  for cam in cams:
+    cam['obj'].ssd_to_camera()
+  # get raw partitions
 for cam in cams:
-  d = cam['obj'].first_found_raw_partition_name()
+  d = cam['obj'].first_found_raw_partition_name() # ssh
   if d!="undefined":
     dirs.append(d)
     print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)
   else:
     cam['disable']=1
     print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)
-    #raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")
-
-# no need
-cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]
-
-# switch ssd to pc for all (not caring which cable is in)
-for cam in cams:
-  cam['obj'].ssd_to_pc()
+      #raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")
+  # no need
+  cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]
+  
+if (not args.lan):
+  # switch ssd to pc for all (not caring which cable is in)
+  for cam in cams:
+    cam['obj'].ssd_to_pc()
 
 # download
 
 plist = []
 all_downloaded = False
 
-for i in range(len(cams)):
-  raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
+#for i in range(len(cams)):
+for i,cam in enumerate(cams):
+#  raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
+  if (args.lan):
+    print(bcolors.OKGREEN+"Initiating data transfer over LAN ..."+bcolors.ENDC)
+  else:    
+    input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
   proceed_to_next = False
   t = 0
   while not all_downloaded:
-    plist = pc.list_partitions()
+    if (args.lan):
+      plist = cam['obj'].list_partitions_as_host() #  Camera partitions, same format as for host (label, path)
+      #[['NT-1TB_2242_0007988000104', 'sda'], ['NT-1TB_2242_0007988000104-part1', 'sda1'], ['NT-1TB_2242_0007988000104-part2', 'sda2']]
+    else:
+      plist = pc.list_partitions() #  HOST(!) partitions, such as /dev/sdc1
+      #[[u'NT-1TB_2242_0007988000104', u'sdc'], [u'NT-1TB_2242_0007988000104-part1', u'sdc1'], [u'NT-1TB_2242_0007988000104-part2', u'sdc2']]
     for d in dirs:
       for p in plist:
         if d==p[0]:
           # p[1] == sdb2
           # hardcoded /dev/sd?1
           if args.n==0:
-            data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
-            data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
+            if (args.lan):
+#              data_size_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_end)
+#              data_skip_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_start)
+              data_size_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_end)
+              data_skip_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_start)
+              #read_camogm_disk_file_blocks
+            else:
+              data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
+              data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
             data_size_blocks -= data_skip_blocks # before it included skipped !
           else:
             data_size_blocks = args.n
@@ -158,34 +181,31 @@ for i in range(len(cams)):
           block_size=    512 # 4096
           data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)
           file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter
-          print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))  
-          pc.download_blocks(args.dest, #  def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512):
-                             "/dev/"+p[1], 
-                             blocks_load=data_size_blocks,
-                             blocks_skip= data_skip_blocks,
-                             file_gb=file_gb,
-                             chunk_blocks=chunk_blocks,
-                             block_size=block_size)
-          '''
-          data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1")
-          data_size = round(data_size,2)
-          # bs is in kB
-          chunk_size = float(args.bs*args.bc)/1024
-          n_chunks = int(math.ceil(data_size/chunk_size))
-
-          if args.n==0:
-            args.n = n_chunks - args.skip
-
-          print("Data size: "+str(data_size)+" GB")
-          print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC)
-          pc.download       (args.dest, "/dev/"+p[1], args.bs,args.bc,args.skip,args.n)
-          '''
-#  def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
-#parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
-#parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
+          print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))
+          if (args.lan):
+            print(cam)  
+            pc.download_blocks_lan(
+                               cam,
+                               args.dest,
+                               "/dev/"+p[1], 
+                               blocks_load=data_size_blocks,
+                               blocks_skip= data_skip_blocks,
+                               file_gb=file_gb,
+                               chunk_blocks=chunk_blocks,
+                               block_size=block_size)
+          else:
+            pc.download_blocks( # after testing use download_blocks_lan (rename/delete download_blocks) with cam=None
+                               args.dest,
+                               "/dev/"+p[1], 
+                               blocks_load=data_size_blocks,
+                               blocks_skip= data_skip_blocks,
+                               file_gb=file_gb,
+                               chunk_blocks=chunk_blocks,
+                               block_size=block_size)
+                
           dirs.remove(d)
           proceed_to_next = True
-          if len(dirs)!=0:
+          if len(dirs) != 0:
             print("wait for the next ssd")
           else:
             all_downloaded = True

x393.py

@@ -11,6 +11,7 @@ import re
 import subprocess
 import time
 import tempfile
+import shutil
 
 def shout(cmd):
   #subprocess.call prints to console
@@ -39,6 +40,7 @@ class Camera:
     self.user = user
     self.ip = ip
     self.sshcmd = "ssh "+user+"@"+ip
+    self.scpcmd = "scp "+user+"@"+ip+":"
     self.disable = False
     self.pattern = "ata-"
     self.check_connection()
@@ -83,6 +85,18 @@ class Camera:
           plist.append(s0.group(0))
     return plist
   
+  def list_partitions_as_host(self):
+    res = shout(self.sshcmd+" 'ls /dev/disk/by-id/ -all ' | grep '"+self.pattern+"'")
+    plist = []
+    for line in res.splitlines():
+      items = line.split(" ")
+      for name in items:
+        if name[0:len(self.pattern)]==self.pattern:
+          item = items[-1].split("/")
+          plist.append([name[len(self.pattern):],item[-1]])
+    return plist
+  
+  
   def list_mounted_partitions(self):
     res = shout(self.sshcmd+" 'df -h'")
     df_list = []
@@ -156,6 +170,26 @@ class Camera:
     time.sleep(1)
     shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")
 
+  def read_camogm_disk_file_blocks(self, part,fname="camogm.disk"):
+    result = 0
+    tmp_mount_point = tempfile.mkdtemp()
+    print(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
+    shout(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
+    try:
+      with open (tmp_mount_point+"/"+fname, "r") as myfile:
+        data=myfile.readlines()
+        if len(data)==2:
+          l2 = data[1]
+          pointers = l2.split("\t")
+          pntr1 = int(pointers[1])
+          pntr2 = int(pointers[2])
+          result = pntr2-pntr1
+    except IOError:
+      print(tmp_mount_point+"/"+fname+" NOT FOUND")
+#    os.rmdir(tmp_mount_point)
+    shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only    
+    return result
+
 class PC():
   def __init__(self):
     self.pattern = "ata-"
@@ -174,9 +208,7 @@ class PC():
   # mounts partition (/dev/sd?1), reads camogm.disk file
   # returns the download size from raw partition ((/dev/sd?2))
   def read_camogm_disk_file(self,part,fname="camogm.disk"):
-
     result = 0
-
     tmp_mount_point = tempfile.mkdtemp()
     print("mounting "+part+" to "+tmp_mount_point)
     shout("sudo mount "+part+" "+tmp_mount_point)
@@ -190,15 +222,34 @@ class PC():
           pntr1 = int(pointers[1])
           pntr2 = int(pointers[2])
           result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
-#          result = float(pntr2-pntr1)*4096/1024/1024/1024 # blocks are now 4096, not 512 bytes!
     except IOError:
       print(tmp_mount_point+"/"+fname+" NOT FOUND")
-
     shout("sudo umount "+tmp_mount_point)
     os.rmdir(tmp_mount_point)
+    return result
 
+#similar as read_camogm_disk_file(), but uses LAN istead of eSATA. Needs additional argument ip
+  def read_camogm_disk_file_lan(self, ip, part, fname="camogm.disk"):
+    result = 0
+    tmp_mount_point = tempfile.mkdtemp()
+    print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
+    shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
+    try:
+      with open (tmp_mount_point+"/"+fname, "r") as myfile:
+        data=myfile.readlines()
+        if len(data)==2:
+          l2 = data[1]
+          pointers = l2.split("\t")
+          pntr1 = int(pointers[1])
+          pntr2 = int(pointers[2])
+          result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
+    except IOError:
+      print(tmp_mount_point+"/"+fname+" NOT FOUND")
+#    os.rmdir(tmp_mount_point)
+    shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only    
     return result
 
+
   """
   Returns block offset from the start of partition to the current LBA pointer
   """
@@ -207,7 +258,6 @@ class PC():
     tmp_mount_point = tempfile.mkdtemp()
     print("mounting "+part+" to "+tmp_mount_point)
     shout("sudo mount "+part+" "+tmp_mount_point)
-
     try:
       with open (tmp_mount_point+"/"+fname, "r") as myfile:
         data=myfile.readlines()
@@ -221,12 +271,30 @@ class PC():
           result = pntr2-pntr1
     except IOError:
       print(tmp_mount_point+"/"+fname+" NOT FOUND")
-
     shout("sudo umount "+tmp_mount_point)
     os.rmdir(tmp_mount_point)
-
     return result
 
+  def read_camogm_disk_file_blocks_lan(self, ip, part,fname="camogm.disk"):
+    result = 0
+    tmp_mount_point = tempfile.mkdtemp()
+    print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
+    shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
+    try:
+      with open (tmp_mount_point+"/"+fname, "r") as myfile:
+        data=myfile.readlines()
+        if len(data)==2:
+          l2 = data[1]
+          pointers = l2.split("\t")
+          pntr1 = int(pointers[1])
+          pntr2 = int(pointers[2])
+#          result = float(pntr2-pntr1)*512/1024/1024/1024
+          result = pntr2-pntr1
+    except IOError:
+      print(tmp_mount_point+"/"+fname+" NOT FOUND")
+#    os.rmdir(tmp_mount_point)
+    shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only    
+    return result
 
   def is_raw(self,part):
     res = shout("sudo blkid | grep "+str(part))
@@ -266,7 +334,6 @@ class PC():
         if i>=dl_skip:
           shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))
       
-#  def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=4096, block_size=4096):
   def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
     chunk_bytes =  block_size * chunk_blocks 
     file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes
@@ -313,8 +380,75 @@ class PC():
         fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
         print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
         shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
-    
-    
+
+#time ssh root@192.168.0.41 "dd if=/dev/sda2 bs=16777216 count=409 skip=322" | dd of=/home/elphel/lwir16-proc/test_dd/file_0001.img
+#     res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id | grep '"+self.pattern+"' | grep '"+partition[-4:]+"''")
+
+  def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
+    chunk_bytes =  block_size * chunk_blocks 
+    file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes
+    if (cam==None):
+        self.is_raw(part)
+        ip = None
+    else:
+        print("TODO: Implement raw partition check over SSH")
+#        print(cam)
+        ip = cam["ip"]    
+    print("Getting raw partition data from "+part)
+    if not os.path.isdir(dest):
+      os.mkdir(dest)
+    dirname = self.partname(part)
+    if dirname!="":
+      dirname = dest+"/"+dirname
+      if not os.path.isdir(dirname):
+        os.mkdir(dirname)
+      num_file = 0
+      # optional first file to align skip to chunk_blocks, 1 block at a time
+      if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):
+        bwrite = chunk_blocks - (blocks_skip % chunk_blocks)
+        if (bwrite > blocks_load):
+          bwrite = blocks_load
+        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
+        print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
+        if (cam == None):
+          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
+        else:
+          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+          shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+        blocks_skip += bwrite
+        blocks_load -= bwrite
+        num_file += 1
+      # write bulk of the blocks, <= file_chunks of chunks in each file
+      while ((blocks_load // chunk_blocks)  > 0):
+        chunks_write = blocks_load // chunk_blocks
+        chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
+        if (chunks_write > file_chunks):
+          chunks_write = file_chunks
+#        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
+        print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
+        if (ip == None):
+          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
+        else:
+          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)
+          shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)
+        bwrite = chunks_write * chunk_blocks
+        blocks_skip += bwrite
+        blocks_load -= bwrite
+        num_file += 1
+      # optionally write the remainder (< chunk), 1 block at a time
+      if (blocks_load > 0):
+#        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
+        print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
+        if (ip == None):
+          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
+        else:  
+          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+          shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
   def partname(self,partition):
     cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"
     res = shout(cmd)