Commit 2a6df399 authored by Andrey Filippov's avatar Andrey Filippov
Browse files

implemented LAN xfer with new version of imgsrv

parent 3ff00693

int_ssd_download.py

+3 −2
Original line number Diff line number Diff line
@@ -224,6 +224,7 @@ for i,cam in enumerate(cams): 
    #cam['rp_name'].ssd_to_pc()



# ssd to camera for all

if (not args.lan):

  for cam in cams:

    cam['obj'].ssd_to_camera()

x393.py

+87 −2
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ import subprocess 
import time

import tempfile

import shutil

import requests



def shout(cmd):

  #subprocess.call prints to console
@@ -41,6 +42,7 @@ class Camera: 
    self.ip = ip

    self.sshcmd = "ssh "+user+"@"+ip

    self.scpcmd = "scp "+user+"@"+ip+":"

    self.lanurl = "http://"+ip+":2323/ssd"

    self.disable = False

    self.pattern = "ata-"

    self.check_connection()
@@ -383,8 +385,8 @@ class PC(): 


#time ssh root@192.168.0.41 "dd if=/dev/sda2 bs=16777216 count=409 skip=322" | dd of=/home/elphel/lwir16-proc/test_dd/file_0001.img

#     res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id | grep '"+self.pattern+"' | grep '"+partition[-4:]+"''")



  def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):

#slow, replaced by download_blocks_lan using modified imgsrv 2023-08-02

  def download_blocks_ssh(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):

    chunk_bytes =  block_size * chunk_blocks 

    file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes

    if (cam==None):
@@ -449,6 +451,89 @@ class PC(): 
        else:  

          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)

          shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)



  def download_with_imgsrv(self, cam, out_file, offs, count ):

    #    self.lanurl = "http://"+ip+":2323/ssd"

    url= cam['obj'].lanurl+str(offs)+":"+str(count)

    print("url="+url)

    with requests.get(url, stream=True) as r:

        r.raise_for_status()

        with open(out_file, 'wb') as f:

            for chunk in r.iter_content(chunk_size=8192): 

                # If you have chunk encoded response uncomment if

                # and set chunk_size parameter to None.

                #if chunk: 

                f.write(chunk)



  def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):

    chunk_bytes =  block_size * chunk_blocks 

    file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes

    if (cam==None):

        self.is_raw(part)

        ip = None

    else:

        print("TODO: Implement raw partition check over SSH")

#        print(cam)

        ip = cam["ip"]    

    print("Getting raw partition data from "+part)

    if not os.path.isdir(dest):

      os.mkdir(dest)

    dirname = self.partname(part)

    if dirname!="":

      dirname = dest+"/"+dirname

      if not os.path.isdir(dirname):

        os.mkdir(dirname)

      num_file = 0

      # optional first file to align skip to chunk_blocks, 1 block at a time

      if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):

        bwrite = chunk_blocks - (blocks_skip % chunk_blocks)

        if (bwrite > blocks_load):

          bwrite = blocks_load

        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"

        print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))

        if (cam == None):

          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))

          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))

        else:

#          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)

#          shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)

          self.download_with_imgsrv(cam, fname, blocks_skip * block_size, bwrite * block_size )          

        blocks_skip += bwrite

        blocks_load -= bwrite

        num_file += 1

      # write bulk of the blocks, <= file_chunks of chunks in each file

      while ((blocks_load // chunk_blocks)  > 0):

        chunks_write = blocks_load // chunk_blocks

        chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks

        if (chunks_write > file_chunks):

          chunks_write = file_chunks

#        fname = dirname+"/"+"file_"+str(num_file)+".img"

        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"

        print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))

        if (ip == None):

          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))

          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))

        else:

#         print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)

#         shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)

          self.download_with_imgsrv(cam, fname, chunks_skip * chunk_bytes, chunks_write * chunk_bytes )          

        bwrite = chunks_write * chunk_blocks

        blocks_skip += bwrite

        blocks_load -= bwrite

        num_file += 1

      # optionally write the remainder (< chunk), 1 block at a time

      if (blocks_load > 0):

#        fname = dirname+"/"+"file_"+str(num_file)+".img"

        fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"

        print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))

        if (ip == None):

          print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))

          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))

        else:  

#          print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)

#         shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)

          self.download_with_imgsrv(cam, fname, blocks_skip * block_size, blocks_load * block_size )          

          

  def partname(self,partition):

    cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"

    res = shout(cmd)