implemented LAN xfer with new version of imgsrv

int_ssd_download.py

@@ -224,7 +224,8 @@ for i,cam in enumerate(cams):
     #cam['rp_name'].ssd_to_pc()
 
 # ssd to camera for all
-for cam in cams:
-  cam['obj'].ssd_to_camera()
+if (not args.lan):
+  for cam in cams:
+    cam['obj'].ssd_to_camera()
 
 print("Done")

x393.py

@@ -12,6 +12,7 @@ import subprocess
 import time
 import tempfile
 import shutil
+import requests
 
 def shout(cmd):
   #subprocess.call prints to console
@@ -41,6 +42,7 @@ class Camera:
     self.ip = ip
     self.sshcmd = "ssh "+user+"@"+ip
     self.scpcmd = "scp "+user+"@"+ip+":"
+    self.lanurl = "http://"+ip+":2323/ssd"
     self.disable = False
     self.pattern = "ata-"
     self.check_connection()
@@ -383,8 +385,8 @@ class PC():
 
 #time ssh root@192.168.0.41 "dd if=/dev/sda2 bs=16777216 count=409 skip=322" | dd of=/home/elphel/lwir16-proc/test_dd/file_0001.img
 #     res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id | grep '"+self.pattern+"' | grep '"+partition[-4:]+"''")
-
-  def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
+#slow, replaced by download_blocks_lan using modified imgsrv 2023-08-02
+  def download_blocks_ssh(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
     chunk_bytes =  block_size * chunk_blocks 
     file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes
     if (cam==None):
@@ -449,6 +451,89 @@ class PC():
         else:  
           print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
           shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+
+  def download_with_imgsrv(self, cam, out_file, offs, count ):
+    #    self.lanurl = "http://"+ip+":2323/ssd"
+    url= cam['obj'].lanurl+str(offs)+":"+str(count)
+    print("url="+url)
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(out_file, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192): 
+                # If you have chunk encoded response uncomment if
+                # and set chunk_size parameter to None.
+                #if chunk: 
+                f.write(chunk)
+
+  def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
+    chunk_bytes =  block_size * chunk_blocks 
+    file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes
+    if (cam==None):
+        self.is_raw(part)
+        ip = None
+    else:
+        print("TODO: Implement raw partition check over SSH")
+#        print(cam)
+        ip = cam["ip"]    
+    print("Getting raw partition data from "+part)
+    if not os.path.isdir(dest):
+      os.mkdir(dest)
+    dirname = self.partname(part)
+    if dirname!="":
+      dirname = dest+"/"+dirname
+      if not os.path.isdir(dirname):
+        os.mkdir(dirname)
+      num_file = 0
+      # optional first file to align skip to chunk_blocks, 1 block at a time
+      if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):
+        bwrite = chunk_blocks - (blocks_skip % chunk_blocks)
+        if (bwrite > blocks_load):
+          bwrite = blocks_load
+        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
+        print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
+        if (cam == None):
+          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
+        else:
+#          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+#          shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+          self.download_with_imgsrv(cam, fname, blocks_skip * block_size, bwrite * block_size )          
+        blocks_skip += bwrite
+        blocks_load -= bwrite
+        num_file += 1
+      # write bulk of the blocks, <= file_chunks of chunks in each file
+      while ((blocks_load // chunk_blocks)  > 0):
+        chunks_write = blocks_load // chunk_blocks
+        chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
+        if (chunks_write > file_chunks):
+          chunks_write = file_chunks
+#        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
+        print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
+        if (ip == None):
+          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
+        else:
+#         print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)
+#         shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)
+          self.download_with_imgsrv(cam, fname, chunks_skip * chunk_bytes, chunks_write * chunk_bytes )          
+        bwrite = chunks_write * chunk_blocks
+        blocks_skip += bwrite
+        blocks_load -= bwrite
+        num_file += 1
+      # optionally write the remainder (< chunk), 1 block at a time
+      if (blocks_load > 0):
+#        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
+        print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
+        if (ip == None):
+          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
+        else:  
+#          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+#         shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
+          self.download_with_imgsrv(cam, fname, blocks_skip * block_size, blocks_load * block_size )          
+          
   def partname(self,partition):
     cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"
     res = shout(cmd)