implemented transfer over ssh (10x slower), restored input(), forced python3 (3ff00693) · Commits · Elphel / elphel-tools-x393

int_ssd_download.py

+65 −45

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		#!/usr/bin/env python3

		# Download footage from ssd's raw partition:
		# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off
		@@ -72,6 +72,7 @@ parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [b
		parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")
		parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")
		parser.add_argument("dest",help="desitnation directory: /data/footage/test")
		parser.add_argument("-l","--lan", action='store_true', help="Transfer data over LAN instead of eSATA")

		args = parser.parse_args()
		print (args)
		@@ -106,13 +107,18 @@ if len(cams)==0:

		pc = x393.PC()



		if (args.lan):
		print ("Using data transfer over LAN")
		else:
		print ("Using data transfer over eSATA, multiplexing camera SSD to eSATA external port")
		# ssd to camera for all
		for cam in cams:
		cam['obj'].ssd_to_camera()

		# get raw partitions
		for cam in cams:
		d = cam['obj'].first_found_raw_partition_name()
		d = cam['obj'].first_found_raw_partition_name() # ssh
		if d!="undefined":
		dirs.append(d)
		print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)
		@@ -120,10 +126,10 @@ for cam in cams:
		cam['disable']=1
		print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)
		#raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")

		# no need
		cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]

		if (not args.lan):
		# switch ssd to pc for all (not caring which cable is in)
		for cam in cams:
		cam['obj'].ssd_to_pc()
		@@ -133,18 +139,35 @@ for cam in cams:
		plist = []
		all_downloaded = False

		for i in range(len(cams)):
		raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
		#for i in range(len(cams)):
		for i,cam in enumerate(cams):
		# raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
		if (args.lan):
		print(bcolors.OKGREEN+"Initiating data transfer over LAN ..."+bcolors.ENDC)
		else:
		input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
		proceed_to_next = False
		t = 0
		while not all_downloaded:
		plist = pc.list_partitions()
		if (args.lan):
		plist = cam['obj'].list_partitions_as_host() # Camera partitions, same format as for host (label, path)
		#[['NT-1TB_2242_0007988000104', 'sda'], ['NT-1TB_2242_0007988000104-part1', 'sda1'], ['NT-1TB_2242_0007988000104-part2', 'sda2']]
		else:
		plist = pc.list_partitions() # HOST(!) partitions, such as /dev/sdc1
		#[[u'NT-1TB_2242_0007988000104', u'sdc'], [u'NT-1TB_2242_0007988000104-part1', u'sdc1'], [u'NT-1TB_2242_0007988000104-part2', u'sdc2']]
		for d in dirs:
		for p in plist:
		if d==p[0]:
		# p[1] == sdb2
		# hardcoded /dev/sd?1
		if args.n==0:
		if (args.lan):
		# data_size_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_end)
		# data_skip_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_start)
		data_size_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_end)
		data_skip_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_start)
		#read_camogm_disk_file_blocks
		else:
		data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
		data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
		data_size_blocks -= data_skip_blocks # before it included skipped !
		@@ -159,30 +182,27 @@ for i in range(len(cams)):
		data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)
		file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter
		print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))
		pc.download_blocks(args.dest, # def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512):
		if (args.lan):
		print(cam)
		pc.download_blocks_lan(
		cam,
		args.dest,
		"/dev/"+p[1],
		blocks_load=data_size_blocks,
		blocks_skip= data_skip_blocks,
		file_gb=file_gb,
		chunk_blocks=chunk_blocks,
		block_size=block_size)
		else:
		pc.download_blocks( # after testing use download_blocks_lan (rename/delete download_blocks) with cam=None
		args.dest,
		"/dev/"+p[1],
		blocks_load=data_size_blocks,
		blocks_skip= data_skip_blocks,
		file_gb=file_gb,
		chunk_blocks=chunk_blocks,
		block_size=block_size)
		'''
		data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1")
		data_size = round(data_size,2)
		# bs is in kB
		chunk_size = float(args.bs*args.bc)/1024
		n_chunks = int(math.ceil(data_size/chunk_size))

		if args.n==0:
		args.n = n_chunks - args.skip

		print("Data size: "+str(data_size)+" GB")
		print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC)
		pc.download (args.dest, "/dev/"+p[1], args.bs,args.bc,args.skip,args.n)
		'''
		# def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
		#parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
		#parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
		dirs.remove(d)
		proceed_to_next = True
		if len(dirs) != 0:

x393.py

+144 −10

Original line number	Diff line number	Diff line
		@@ -11,6 +11,7 @@ import re
		import subprocess
		import time
		import tempfile
		import shutil

		def shout(cmd):
		#subprocess.call prints to console
		@@ -39,6 +40,7 @@ class Camera:
		self.user = user
		self.ip = ip
		self.sshcmd = "ssh "+user+"@"+ip
		self.scpcmd = "scp "+user+"@"+ip+":"
		self.disable = False
		self.pattern = "ata-"
		self.check_connection()
		@@ -83,6 +85,18 @@ class Camera:
		plist.append(s0.group(0))
		return plist

		def list_partitions_as_host(self):
		res = shout(self.sshcmd+" 'ls /dev/disk/by-id/ -all ' \| grep '"+self.pattern+"'")
		plist = []
		for line in res.splitlines():
		items = line.split(" ")
		for name in items:
		if name[0:len(self.pattern)]==self.pattern:
		item = items[-1].split("/")
		plist.append([name[len(self.pattern):],item[-1]])
		return plist


		def list_mounted_partitions(self):
		res = shout(self.sshcmd+" 'df -h'")
		df_list = []
		@@ -156,6 +170,26 @@ class Camera:
		time.sleep(1)
		shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")

		def read_camogm_disk_file_blocks(self, part,fname="camogm.disk"):
		result = 0
		tmp_mount_point = tempfile.mkdtemp()
		print(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
		shout(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
		try:
		with open (tmp_mount_point+"/"+fname, "r") as myfile:
		data=myfile.readlines()
		if len(data)==2:
		l2 = data[1]
		pointers = l2.split("\t")
		pntr1 = int(pointers[1])
		pntr2 = int(pointers[2])
		result = pntr2-pntr1
		except IOError:
		print(tmp_mount_point+"/"+fname+" NOT FOUND")
		# os.rmdir(tmp_mount_point)
		shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
		return result

		class PC():
		def __init__(self):
		self.pattern = "ata-"
		@@ -174,9 +208,7 @@ class PC():
		# mounts partition (/dev/sd?1), reads camogm.disk file
		# returns the download size from raw partition ((/dev/sd?2))
		def read_camogm_disk_file(self,part,fname="camogm.disk"):

		result = 0

		tmp_mount_point = tempfile.mkdtemp()
		print("mounting "+part+" to "+tmp_mount_point)
		shout("sudo mount "+part+" "+tmp_mount_point)
		@@ -190,15 +222,34 @@ class PC():
		pntr1 = int(pointers[1])
		pntr2 = int(pointers[2])
		result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
		# result = float(pntr2-pntr1)*4096/1024/1024/1024 # blocks are now 4096, not 512 bytes!
		except IOError:
		print(tmp_mount_point+"/"+fname+" NOT FOUND")

		shout("sudo umount "+tmp_mount_point)
		os.rmdir(tmp_mount_point)
		return result

		#similar as read_camogm_disk_file(), but uses LAN istead of eSATA. Needs additional argument ip
		def read_camogm_disk_file_lan(self, ip, part, fname="camogm.disk"):
		result = 0
		tmp_mount_point = tempfile.mkdtemp()
		print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
		shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
		try:
		with open (tmp_mount_point+"/"+fname, "r") as myfile:
		data=myfile.readlines()
		if len(data)==2:
		l2 = data[1]
		pointers = l2.split("\t")
		pntr1 = int(pointers[1])
		pntr2 = int(pointers[2])
		result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
		except IOError:
		print(tmp_mount_point+"/"+fname+" NOT FOUND")
		# os.rmdir(tmp_mount_point)
		shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
		return result


		"""
		Returns block offset from the start of partition to the current LBA pointer
		"""
		@@ -207,7 +258,6 @@ class PC():
		tmp_mount_point = tempfile.mkdtemp()
		print("mounting "+part+" to "+tmp_mount_point)
		shout("sudo mount "+part+" "+tmp_mount_point)

		try:
		with open (tmp_mount_point+"/"+fname, "r") as myfile:
		data=myfile.readlines()
		@@ -221,12 +271,30 @@ class PC():
		result = pntr2-pntr1
		except IOError:
		print(tmp_mount_point+"/"+fname+" NOT FOUND")

		shout("sudo umount "+tmp_mount_point)
		os.rmdir(tmp_mount_point)

		return result

		def read_camogm_disk_file_blocks_lan(self, ip, part,fname="camogm.disk"):
		result = 0
		tmp_mount_point = tempfile.mkdtemp()
		print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
		shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
		try:
		with open (tmp_mount_point+"/"+fname, "r") as myfile:
		data=myfile.readlines()
		if len(data)==2:
		l2 = data[1]
		pointers = l2.split("\t")
		pntr1 = int(pointers[1])
		pntr2 = int(pointers[2])
		# result = float(pntr2-pntr1)*512/1024/1024/1024
		result = pntr2-pntr1
		except IOError:
		print(tmp_mount_point+"/"+fname+" NOT FOUND")
		# os.rmdir(tmp_mount_point)
		shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
		return result

		def is_raw(self,part):
		res = shout("sudo blkid \| grep "+str(part))
		@@ -266,7 +334,6 @@ class PC():
		if i>=dl_skip:
		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))

		# def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=4096, block_size=4096):
		def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
		chunk_bytes = block_size * chunk_blocks
		file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
		@@ -314,7 +381,74 @@ class PC():
		print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))

		#time ssh root@192.168.0.41 "dd if=/dev/sda2 bs=16777216 count=409 skip=322" \| dd of=/home/elphel/lwir16-proc/test_dd/file_0001.img
		# res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id \| grep '"+self.pattern+"' \| grep '"+partition[-4:]+"''")

		def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
		chunk_bytes = block_size * chunk_blocks
		file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
		if (cam==None):
		self.is_raw(part)
		ip = None
		else:
		print("TODO: Implement raw partition check over SSH")
		# print(cam)
		ip = cam["ip"]
		print("Getting raw partition data from "+part)
		if not os.path.isdir(dest):
		os.mkdir(dest)
		dirname = self.partname(part)
		if dirname!="":
		dirname = dest+"/"+dirname
		if not os.path.isdir(dirname):
		os.mkdir(dirname)
		num_file = 0
		# optional first file to align skip to chunk_blocks, 1 block at a time
		if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):
		bwrite = chunk_blocks - (blocks_skip % chunk_blocks)
		if (bwrite > blocks_load):
		bwrite = blocks_load
		fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
		print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
		if (cam == None):
		print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
		else:
		print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
		shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
		blocks_skip += bwrite
		blocks_load -= bwrite
		num_file += 1
		# write bulk of the blocks, <= file_chunks of chunks in each file
		while ((blocks_load // chunk_blocks) > 0):
		chunks_write = blocks_load // chunk_blocks
		chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
		if (chunks_write > file_chunks):
		chunks_write = file_chunks
		# fname = dirname+"/"+"file_"+str(num_file)+".img"
		fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
		print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
		if (ip == None):
		print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
		else:
		print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' \| dd of="+fname)
		shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' \| dd of="+fname)
		bwrite = chunks_write * chunk_blocks
		blocks_skip += bwrite
		blocks_load -= bwrite
		num_file += 1
		# optionally write the remainder (< chunk), 1 block at a time
		if (blocks_load > 0):
		# fname = dirname+"/"+"file_"+str(num_file)+".img"
		fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
		print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
		if (ip == None):
		print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
		else:
		print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
		shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
		def partname(self,partition):
		cmd = "ls /dev/disk/by-id/ -all \| grep '"+self.pattern+"' \| grep '"+partition[-4:]+"'"
		res = shout(cmd)

Admin message