implemented transfer over ssh (10x slower), restored input(), forced python3 (3ff00693) · Commits · Elphel / elphel-tools-x393

int_ssd_download.py

+65 −45

Original line number	Original line	Diff line number	Diff line
	#!/usr/bin/env python		#!/usr/bin/env python3

	# Download footage from ssd's raw partition:		# Download footage from ssd's raw partition:
	# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off		# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off
	@@ -72,6 +72,7 @@ parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [b
	parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")		parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")
	parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")		parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")
	parser.add_argument("dest",help="desitnation directory: /data/footage/test")		parser.add_argument("dest",help="desitnation directory: /data/footage/test")
			parser.add_argument("-l","--lan", action='store_true', help="Transfer data over LAN instead of eSATA")

	args = parser.parse_args()		args = parser.parse_args()
	print (args)		print (args)
	@@ -106,13 +107,18 @@ if len(cams)==0:

	pc = x393.PC()		pc = x393.PC()



			if (args.lan):
			print ("Using data transfer over LAN")
			else:
			print ("Using data transfer over eSATA, multiplexing camera SSD to eSATA external port")
	# ssd to camera for all		# ssd to camera for all
	for cam in cams:		for cam in cams:
	cam['obj'].ssd_to_camera()		cam['obj'].ssd_to_camera()

	# get raw partitions		# get raw partitions
	for cam in cams:		for cam in cams:
	d = cam['obj'].first_found_raw_partition_name()		d = cam['obj'].first_found_raw_partition_name() # ssh
	if d!="undefined":		if d!="undefined":
	dirs.append(d)		dirs.append(d)
	print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)		print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)
	@@ -120,10 +126,10 @@ for cam in cams:
	cam['disable']=1		cam['disable']=1
	print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)		print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)
	#raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")		#raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")

	# no need		# no need
	cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]		cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]

			if (not args.lan):
	# switch ssd to pc for all (not caring which cable is in)		# switch ssd to pc for all (not caring which cable is in)
	for cam in cams:		for cam in cams:
	cam['obj'].ssd_to_pc()		cam['obj'].ssd_to_pc()
	@@ -133,18 +139,35 @@ for cam in cams:
	plist = []		plist = []
	all_downloaded = False		all_downloaded = False

	for i in range(len(cams)):		#for i in range(len(cams)):
	raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)		for i,cam in enumerate(cams):
			# raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
			if (args.lan):
			print(bcolors.OKGREEN+"Initiating data transfer over LAN ..."+bcolors.ENDC)
			else:
			input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
	proceed_to_next = False		proceed_to_next = False
	t = 0		t = 0
	while not all_downloaded:		while not all_downloaded:
	plist = pc.list_partitions()		if (args.lan):
			plist = cam['obj'].list_partitions_as_host() # Camera partitions, same format as for host (label, path)
			#[['NT-1TB_2242_0007988000104', 'sda'], ['NT-1TB_2242_0007988000104-part1', 'sda1'], ['NT-1TB_2242_0007988000104-part2', 'sda2']]
			else:
			plist = pc.list_partitions() # HOST(!) partitions, such as /dev/sdc1
			#[[u'NT-1TB_2242_0007988000104', u'sdc'], [u'NT-1TB_2242_0007988000104-part1', u'sdc1'], [u'NT-1TB_2242_0007988000104-part2', u'sdc2']]
	for d in dirs:		for d in dirs:
	for p in plist:		for p in plist:
	if d==p[0]:		if d==p[0]:
	# p[1] == sdb2		# p[1] == sdb2
	# hardcoded /dev/sd?1		# hardcoded /dev/sd?1
	if args.n==0:		if args.n==0:
			if (args.lan):
			# data_size_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_end)
			# data_skip_blocks = pc.read_camogm_disk_file_blocks_lan(cams[i].ip, p[1][0:-1]+"1", args.file_start)
			data_size_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_end)
			data_skip_blocks = cam['obj'].read_camogm_disk_file_blocks(p[1][0:-1]+"1", args.file_start)
			#read_camogm_disk_file_blocks
			else:
	data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)		data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
	data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)		data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
	data_size_blocks -= data_skip_blocks # before it included skipped !		data_size_blocks -= data_skip_blocks # before it included skipped !
	@@ -159,30 +182,27 @@ for i in range(len(cams)):
	data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)		data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)
	file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter		file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter
	print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))		print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))
	pc.download_blocks(args.dest, # def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512):		if (args.lan):
			print(cam)
			pc.download_blocks_lan(
			cam,
			args.dest,
			"/dev/"+p[1],
			blocks_load=data_size_blocks,
			blocks_skip= data_skip_blocks,
			file_gb=file_gb,
			chunk_blocks=chunk_blocks,
			block_size=block_size)
			else:
			pc.download_blocks( # after testing use download_blocks_lan (rename/delete download_blocks) with cam=None
			args.dest,
	"/dev/"+p[1],		"/dev/"+p[1],
	blocks_load=data_size_blocks,		blocks_load=data_size_blocks,
	blocks_skip= data_skip_blocks,		blocks_skip= data_skip_blocks,
	file_gb=file_gb,		file_gb=file_gb,
	chunk_blocks=chunk_blocks,		chunk_blocks=chunk_blocks,
	block_size=block_size)		block_size=block_size)
	'''
	data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1")
	data_size = round(data_size,2)
	# bs is in kB
	chunk_size = float(args.bs*args.bc)/1024
	n_chunks = int(math.ceil(data_size/chunk_size))

	if args.n==0:
	args.n = n_chunks - args.skip

	print("Data size: "+str(data_size)+" GB")
	print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC)
	pc.download (args.dest, "/dev/"+p[1], args.bs,args.bc,args.skip,args.n)
	'''
	# def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
	#parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
	#parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
	dirs.remove(d)		dirs.remove(d)
	proceed_to_next = True		proceed_to_next = True
	if len(dirs) != 0:		if len(dirs) != 0:

x393.py

+144 −10

Original line number	Original line	Diff line number	Diff line
	@@ -11,6 +11,7 @@ import re
	import subprocess		import subprocess
	import time		import time
	import tempfile		import tempfile
			import shutil

	def shout(cmd):		def shout(cmd):
	#subprocess.call prints to console		#subprocess.call prints to console
	@@ -39,6 +40,7 @@ class Camera:
	self.user = user		self.user = user
	self.ip = ip		self.ip = ip
	self.sshcmd = "ssh "+user+"@"+ip		self.sshcmd = "ssh "+user+"@"+ip
			self.scpcmd = "scp "+user+"@"+ip+":"
	self.disable = False		self.disable = False
	self.pattern = "ata-"		self.pattern = "ata-"
	self.check_connection()		self.check_connection()
	@@ -83,6 +85,18 @@ class Camera:
	plist.append(s0.group(0))		plist.append(s0.group(0))
	return plist		return plist

			def list_partitions_as_host(self):
			res = shout(self.sshcmd+" 'ls /dev/disk/by-id/ -all ' \| grep '"+self.pattern+"'")
			plist = []
			for line in res.splitlines():
			items = line.split(" ")
			for name in items:
			if name[0:len(self.pattern)]==self.pattern:
			item = items[-1].split("/")
			plist.append([name[len(self.pattern):],item[-1]])
			return plist


	def list_mounted_partitions(self):		def list_mounted_partitions(self):
	res = shout(self.sshcmd+" 'df -h'")		res = shout(self.sshcmd+" 'df -h'")
	df_list = []		df_list = []
	@@ -156,6 +170,26 @@ class Camera:
	time.sleep(1)		time.sleep(1)
	shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")		shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")

			def read_camogm_disk_file_blocks(self, part,fname="camogm.disk"):
			result = 0
			tmp_mount_point = tempfile.mkdtemp()
			print(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
			shout(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
			try:
			with open (tmp_mount_point+"/"+fname, "r") as myfile:
			data=myfile.readlines()
			if len(data)==2:
			l2 = data[1]
			pointers = l2.split("\t")
			pntr1 = int(pointers[1])
			pntr2 = int(pointers[2])
			result = pntr2-pntr1
			except IOError:
			print(tmp_mount_point+"/"+fname+" NOT FOUND")
			# os.rmdir(tmp_mount_point)
			shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
			return result

	class PC():		class PC():
	def __init__(self):		def __init__(self):
	self.pattern = "ata-"		self.pattern = "ata-"
	@@ -174,9 +208,7 @@ class PC():
	# mounts partition (/dev/sd?1), reads camogm.disk file		# mounts partition (/dev/sd?1), reads camogm.disk file
	# returns the download size from raw partition ((/dev/sd?2))		# returns the download size from raw partition ((/dev/sd?2))
	def read_camogm_disk_file(self,part,fname="camogm.disk"):		def read_camogm_disk_file(self,part,fname="camogm.disk"):

	result = 0		result = 0

	tmp_mount_point = tempfile.mkdtemp()		tmp_mount_point = tempfile.mkdtemp()
	print("mounting "+part+" to "+tmp_mount_point)		print("mounting "+part+" to "+tmp_mount_point)
	shout("sudo mount "+part+" "+tmp_mount_point)		shout("sudo mount "+part+" "+tmp_mount_point)
	@@ -190,15 +222,34 @@ class PC():
	pntr1 = int(pointers[1])		pntr1 = int(pointers[1])
	pntr2 = int(pointers[2])		pntr2 = int(pointers[2])
	result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size		result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
	# result = float(pntr2-pntr1)*4096/1024/1024/1024 # blocks are now 4096, not 512 bytes!
	except IOError:		except IOError:
	print(tmp_mount_point+"/"+fname+" NOT FOUND")		print(tmp_mount_point+"/"+fname+" NOT FOUND")

	shout("sudo umount "+tmp_mount_point)		shout("sudo umount "+tmp_mount_point)
	os.rmdir(tmp_mount_point)		os.rmdir(tmp_mount_point)
			return result

			#similar as read_camogm_disk_file(), but uses LAN istead of eSATA. Needs additional argument ip
			def read_camogm_disk_file_lan(self, ip, part, fname="camogm.disk"):
			result = 0
			tmp_mount_point = tempfile.mkdtemp()
			print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
			shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
			try:
			with open (tmp_mount_point+"/"+fname, "r") as myfile:
			data=myfile.readlines()
			if len(data)==2:
			l2 = data[1]
			pointers = l2.split("\t")
			pntr1 = int(pointers[1])
			pntr2 = int(pointers[2])
			result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
			except IOError:
			print(tmp_mount_point+"/"+fname+" NOT FOUND")
			# os.rmdir(tmp_mount_point)
			shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
	return result		return result


	"""		"""
	Returns block offset from the start of partition to the current LBA pointer		Returns block offset from the start of partition to the current LBA pointer
	"""		"""
	@@ -207,7 +258,6 @@ class PC():
	tmp_mount_point = tempfile.mkdtemp()		tmp_mount_point = tempfile.mkdtemp()
	print("mounting "+part+" to "+tmp_mount_point)		print("mounting "+part+" to "+tmp_mount_point)
	shout("sudo mount "+part+" "+tmp_mount_point)		shout("sudo mount "+part+" "+tmp_mount_point)

	try:		try:
	with open (tmp_mount_point+"/"+fname, "r") as myfile:		with open (tmp_mount_point+"/"+fname, "r") as myfile:
	data=myfile.readlines()		data=myfile.readlines()
	@@ -221,12 +271,30 @@ class PC():
	result = pntr2-pntr1		result = pntr2-pntr1
	except IOError:		except IOError:
	print(tmp_mount_point+"/"+fname+" NOT FOUND")		print(tmp_mount_point+"/"+fname+" NOT FOUND")

	shout("sudo umount "+tmp_mount_point)		shout("sudo umount "+tmp_mount_point)
	os.rmdir(tmp_mount_point)		os.rmdir(tmp_mount_point)

	return result		return result

			def read_camogm_disk_file_blocks_lan(self, ip, part,fname="camogm.disk"):
			result = 0
			tmp_mount_point = tempfile.mkdtemp()
			print("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
			shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
			try:
			with open (tmp_mount_point+"/"+fname, "r") as myfile:
			data=myfile.readlines()
			if len(data)==2:
			l2 = data[1]
			pointers = l2.split("\t")
			pntr1 = int(pointers[1])
			pntr2 = int(pointers[2])
			# result = float(pntr2-pntr1)*512/1024/1024/1024
			result = pntr2-pntr1
			except IOError:
			print(tmp_mount_point+"/"+fname+" NOT FOUND")
			# os.rmdir(tmp_mount_point)
			shutil.rmtree(tmp_mount_point, ignore_errors=True) # non-empty, may contain read-only
			return result

	def is_raw(self,part):		def is_raw(self,part):
	res = shout("sudo blkid \| grep "+str(part))		res = shout("sudo blkid \| grep "+str(part))
	@@ -266,7 +334,6 @@ class PC():
	if i>=dl_skip:		if i>=dl_skip:
	shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))

	# def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=4096, block_size=4096):
	def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):		def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
	chunk_bytes = block_size * chunk_blocks		chunk_bytes = block_size * chunk_blocks
	file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes		file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
	@@ -314,7 +381,74 @@ class PC():
	print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))		print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
	shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))		shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))

			#time ssh root@192.168.0.41 "dd if=/dev/sda2 bs=16777216 count=409 skip=322" \| dd of=/home/elphel/lwir16-proc/test_dd/file_0001.img
			# res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id \| grep '"+self.pattern+"' \| grep '"+partition[-4:]+"''")

			def download_blocks_lan(self, cam, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
			chunk_bytes = block_size * chunk_blocks
			file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
			if (cam==None):
			self.is_raw(part)
			ip = None
			else:
			print("TODO: Implement raw partition check over SSH")
			# print(cam)
			ip = cam["ip"]
			print("Getting raw partition data from "+part)
			if not os.path.isdir(dest):
			os.mkdir(dest)
			dirname = self.partname(part)
			if dirname!="":
			dirname = dest+"/"+dirname
			if not os.path.isdir(dirname):
			os.mkdir(dirname)
			num_file = 0
			# optional first file to align skip to chunk_blocks, 1 block at a time
			if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):
			bwrite = chunk_blocks - (blocks_skip % chunk_blocks)
			if (bwrite > blocks_load):
			bwrite = blocks_load
			fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
			print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
			if (cam == None):
			print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
			shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
			else:
			print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
			shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
			blocks_skip += bwrite
			blocks_load -= bwrite
			num_file += 1
			# write bulk of the blocks, <= file_chunks of chunks in each file
			while ((blocks_load // chunk_blocks) > 0):
			chunks_write = blocks_load // chunk_blocks
			chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
			if (chunks_write > file_chunks):
			chunks_write = file_chunks
			# fname = dirname+"/"+"file_"+str(num_file)+".img"
			fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
			print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
			if (ip == None):
			print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
			shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
			else:
			print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' \| dd of="+fname)
			shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)+"' \| dd of="+fname)
			bwrite = chunks_write * chunk_blocks
			blocks_skip += bwrite
			blocks_load -= bwrite
			num_file += 1
			# optionally write the remainder (< chunk), 1 block at a time
			if (blocks_load > 0):
			# fname = dirname+"/"+"file_"+str(num_file)+".img"
			fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
			print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
			if (ip == None):
			print("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
			shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
			else:
			print(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
			shout(cam['obj'].sshcmd+" 'dd if="+part+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)+"' \| dd of="+fname)
	def partname(self,partition):		def partname(self,partition):
	cmd = "ls /dev/disk/by-id/ -all \| grep '"+self.pattern+"' \| grep '"+partition[-4:]+"'"		cmd = "ls /dev/disk/by-id/ -all \| grep '"+self.pattern+"' \| grep '"+partition[-4:]+"'"
	res = shout(cmd)		res = shout(cmd)

Admin message