Skip to content

E
elphel-tools-x393
Commit 3ff00693 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

implemented transfer over ssh (10x slower), restored input(), forced python3

parent 5e1a0164
Hide whitespace changes
Inline Side-by-side
Showing with 209 additions and 55 deletions
int_ssd_download.py
View file @ 3ff00693
#!/usr/bin/env python
#!/usr/bin/env python3
# Download footage from ssd's raw partition:
# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off
......@@ -72,6 +72,7 @@ parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [b
parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")
parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")
parser.add_argument("dest",help="desitnation directory: /data/footage/test")
parser.add_argument("-l","--lan", action='store_true', help="Transfer data over LAN instead of eSATA")
args = parser.parse_args()
print (args)
......@@ -106,47 +107,69 @@ if len(cams)==0:
pc = x393.PC()
# ssd to camera for all
for cam in cams:
cam['obj'].ssd_to_camera()
# get raw partitions
if (args.lan):
print ("Using data transfer over LAN")
else:
print ("Using data transfer over eSATA, multiplexing camera SSD to eSATA external port")
# ssd to camera for all
for cam in cams:
cam['obj'].ssd_to_camera()
# get raw partitions
for cam in cams:
d = cam['obj'].first_found_raw_partition_name()
d = cam['obj'].first_found_raw_partition_name() # ssh
if d!="undefined":
dirs.append(d)
print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)
else:
cam['disable']=1
print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)
#raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")
# no need
cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]
# switch ssd to pc for all (not caring which cable is in)
for cam in cams:
cam['obj'].ssd_to_pc()
#raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")
# no need
cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]
if (not args.lan):
# switch ssd to pc for all (not caring which cable is in)
for cam in cams:
cam['obj'].ssd_to_pc()
# download
plist = []
all_downloaded = False
for i in range(len(cams)):
raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
#for i in range(len(cams)):
for i,cam in enumerate(cams):
# raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
if (args.lan):
print(bcolors.OKGREEN+"Initiating data transfer over LAN ..."+bcolors.ENDC)
else:
input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
proceed_to_next = False
t = 0
while not all_downloaded:
plist = pc.list_partitions()
if (args.lan):
plist = cam['obj'].list_partitions_as_host() # Camera partitions, same format as for host (label, path)
#[['NT-1TB_2242_0007988000104', 'sda'], ['NT-1TB_2242_0007988000104-part1', 'sda1'], ['NT-1TB_2242_0007988000104-part2', 'sda2']]
else:
plist = pc.list_partitions() # HOST(!) partitions, such as /dev/sdc1
#[[u'NT-1TB_2242_0007988000104', u'sdc'], [u'NT-1TB_2242_0007988000104-part1', u'sdc1'], [u'NT-1TB_2242_0007988000104-part2', u'sdc2']]
for d in dirs:
for p in plist:
if d==p[0]:
# p[1] == sdb2
# hardcoded /dev/sd?1
if args.n==0:
data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
if (args.lan):
# data_size_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_end)
# data_skip_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_start)
data_size_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_end)
data_skip_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_start)
#read_camogm_disk_file_blocks
else:
data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
data_size_blocks -= data_skip_blocks # before it included skipped !
else:
data_size_blocks = args.n
......@@ -158,34 +181,31 @@ for i in range(len(cams)):
block_size= 512 # 4096
data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)
file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter
print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))
pc.download_blocks(args.dest, # def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512):
"/dev/"+p[1],
blocks_load=data_size_blocks,
blocks_skip= data_skip_blocks,
file_gb=file_gb,
chunk_blocks=chunk_blocks,
block_size=block_size)
'''
data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1")
data_size = round(data_size,2)
# bs is in kB
chunk_size = float(args.bs*args.bc)/1024
n_chunks = int(math.ceil(data_size/chunk_size))
if args.n==0:
args.n = n_chunks - args.skip
print("Data size: "+str(data_size)+" GB")
print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC)
pc.download (args.dest, "/dev/"+p[1], args.bs,args.bc,args.skip,args.n)
'''
# def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
#parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
#parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))
if (args.lan):
print(cam)
pc.download_blocks_lan(
cam,
args.dest,
"/dev/"+p[1],
blocks_load=data_size_blocks,
blocks_skip= data_skip_blocks,
file_gb=file_gb,
chunk_blocks=chunk_blocks,
block_size=block_size)
else:
pc.download_blocks( # after testing use download_blocks_lan (rename/delete download_blocks) with cam=None
args.dest,
"/dev/"+p[1],
blocks_load=data_size_blocks,
blocks_skip= data_skip_blocks,
file_gb=file_gb,
chunk_blocks=chunk_blocks,
block_size=block_size)
dirs.remove(d)
proceed_to_next = True
if len(dirs)!=0:
if len(dirs) != 0:
print("wait for the next ssd")
else:
all_downloaded = True
......
x393.py
View file @ 3ff00693
......@@ -11,6 +11,7 @@ import re
import subprocess
import time
import tempfile
import shutil
def shout(cmd):
#subprocess.call prints to console
......@@ -39,6 +40,7 @@ class Camera:
self.user = user
self.ip = ip
self.sshcmd = "ssh "+user+"@"+ip
self.scpcmd = "scp "+user+"@"+ip+":"
self.disable = False
self.pattern = "ata-"
self.check_connection()
......@@ -83,6 +85,18 @@ class Camera:
plist.append(s0.group(0))
return plist
def list_partitions_as_host(self):
res = shout(self.sshcmd+" 'ls /dev/disk/by-id/ -all ' | grep '"+self.pattern+"'")
plist = []
for line in res.splitlines():
items = line.split(" ")
for name in items:
if name[0:len(self.pattern)]==self.pattern:
item = items[-1].split("/")
plist.append([name[len(self.pattern):],item[-1]])
return plist
def list_mounted_partitions(self):
res = shout(self.sshcmd+" 'df -h'")
df_list = []
......@@ -156,6 +170,26 @@ class Camera:
time.sleep(1)
shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")
def read_camogm_disk_file_blocks(self, part,fname="camogm.disk"):
result = 0
tmp_mount_point = tempfile.mkdtemp()
print(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
shout(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
try:
with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines()
if len(data)==2:
l2 = data[1]
pointers = l2.split("\t")
pntr1 = int(pointers[1])
pntr2 = int(pointers[2])
result = pntr2-pntr1
except IOError:
print(tmp_mount_point+"/"+fname+" NOT FOUND")
# os.rmdir(tmp_mount_point)
shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
return result
class PC():
def __init__(self):
self.pattern = "ata-"
......@@ -174,9 +208,7 @@ class PC():
# mounts partition (/dev/sd?1), reads camogm.disk file
# returns the download size from raw partition ((/dev/sd?2))
def read_camogm_disk_file(self,part,fname="camogm.disk"):
result = 0
tmp_mount_point = tempfile.mkdtemp()
print("mounting "+part+" to "+tmp_mount_point)
shout("sudo mount "+part+" "+tmp_mount_point)
......@@ -190,15 +222,34 @@ class PC():
pntr1 = int(pointers[1])
pntr2 = int(pointers[2])
result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
# result = float(pntr2-pntr1)*4096/1024/1024/1024 # blocks are now 4096, not 512 bytes!
except IOError:
print(tmp_mount_point+"/"+fname+" NOT FOUND")
shout("sudo umount "+tmp_mount_point)
os.rmdir(tmp_mount_point)
return result
#similar as read_camogm_disk_file(), but uses LAN istead of eSATA. Needs additional argument ip
def read_camogm_disk_file_lan(self, ip, part, fname="camogm.disk"):
result = 0
tmp_mount_point = tempfile.mkdtemp()
print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
try:
with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines()
if len(data)==2:
l2 = data[1]
pointers = l2.split("\t")
pntr1 = int(pointers[1])
pntr2 = int(pointers[2])
result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
except IOError:
print(tmp_mount_point+"/"+fname+" NOT FOUND")
# os.rmdir(tmp_mount_point)
shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
return result
"""
Returns block offset from the start of partition to the current LBA pointer
"""
......@@ -207,7 +258,6 @@ class PC():
tmp_mount_point = tempfile.mkdtemp()
print("mounting "+part+" to "+tmp_mount_point)
shout("sudo mount "+part+" "+tmp_mount_point)
try:
with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines()
......@@ -221,12 +271,30 @@ class PC():
result = pntr2-pntr1
except IOError:
print(tmp_mount_point+"/"+fname+" NOT FOUND")
shout("sudo umount "+tmp_mount_point)
os.rmdir(tmp_mount_point)
return result
def read_camogm_disk_file_blocks_lan(self, ip, part,fname="camogm.disk"):
result = 0
tmp_mount_point = tempfile.mkdtemp()
print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
try:
with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines()
if len(data)==2:
l2 = data[1]
pointers = l2.split("\t")
pntr1 = int(pointers[1])
pntr2 = int(pointers[2])
# result = float(pntr2-pntr1)*512/1024/1024/1024
result = pntr2-pntr1
except IOError:
print(tmp_mount_point+"/"+fname+" NOT FOUND")
# os.rmdir(tmp_mount_point)
shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
return result
def is_raw(self,part):
res = shout("sudo blkid | grep "+str(part))
......@@ -266,7 +334,6 @@ class PC():
if i>=dl_skip:
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))
# def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=4096, block_size=4096):
def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
chunk_bytes = block_size * chunk_blocks
file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
......@@ -313,8 +380,75 @@ class PC():
fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
#time ssh root@192.168.0.41 "dd if=/dev/sda2 bs=16777216 count=409 skip=322" | dd of=/home/elphel/lwir16-proc/test_dd/file_0001.img
# res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id | grep '"+self.pattern+"' | grep '"+partition[-4:]+"''")
def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
chunk_bytes = block_size * chunk_blocks
file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
if (cam==None):
self.is_raw(part)
ip = None
else:
print("TODO: Implement raw partition check over SSH")
# print(cam)
ip = cam["ip"]
print("Getting raw partition data from "+part)
if not os.path.isdir(dest):
os.mkdir(dest)
dirname = self.partname(part)
if dirname!="":
dirname = dest+"/"+dirname
if not os.path.isdir(dirname):
os.mkdir(dirname)
num_file = 0
# optional first file to align skip to chunk_blocks, 1 block at a time
if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):
bwrite = chunk_blocks - (blocks_skip % chunk_blocks)
if (bwrite > blocks_load):
bwrite = blocks_load
fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
if (cam == None):
print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
else:
print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)
shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' | dd of="+fname)
blocks_skip += bwrite
blocks_load -= bwrite
num_file += 1
# write bulk of the blocks, <= file_chunks of chunks in each file
while ((blocks_load // chunk_blocks) > 0):
chunks_write = blocks_load // chunk_blocks
chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
if (chunks_write > file_chunks):
chunks_write = file_chunks
# fname = dirname+"/"+"file_"+str(num_file)+".img"
fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
if (ip == None):
print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
else:
print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)
shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' | dd of="+fname)
bwrite = chunks_write * chunk_blocks
blocks_skip += bwrite
blocks_load -= bwrite
num_file += 1
# optionally write the remainder (< chunk), 1 block at a time
if (blocks_load > 0):
# fname = dirname+"/"+"file_"+str(num_file)+".img"
fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
if (ip == None):
print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
else:
print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' | dd of="+fname)
def partname(self,partition):
cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"
res = shout(cmd)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment