initial

ext_ssd_dl_x393.py

+#!/usr/bin/env python
+
+# Download footage from ssd's raw partition:
+# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off
+#   ** All information is in the Exif headers, so no need to match SSDs to subcameras
+# * camera's internal SSD, raw partition, camera is powered on.
+#
+
+__author__ = "Elphel"
+__copyright__ = "Copyright 2016, Elphel, Inc."
+__license__ = "GPL"
+__version__ = "3.0+"
+__maintainer__ = "Oleg K Dzhimiev"
+__email__ = "oleg@elphel.com"
+__status__ = "Development"
+
+import argparse
+import os
+import sys
+import time
+
+import x393
+from x393 import bcolors
+
+usage = """
+
+help:
+
+1. To locate a newly connected SSD device, run:
+
+  $ dmesg
+  
+[] scsi 0:0:0:0: Direct-Access     ATA      Crucial_CT250MX2 MU02 PQ: 0 ANSI: 5
+[] ata1.00: Enabling discard_zeroes_data
+[] sd 0:0:0:0: Attached scsi generic sg0 type 0
+[] sd 0:0:0:0: [sdb] 488397168 512-byte logical blocks: (250 GB/232 GiB)
+[] sd 0:0:0:0: [sdb] 4096-byte physical blocks
+[] sd 0:0:0:0: [sdb] Write Protect is off
+[] sd 0:0:0:0: [sdb] Mode Sense: 00 3a 00 00
+[] sd 0:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
+[] ata1.00: Enabling discard_zeroes_data
+[]  sdb: sdb1 sdb2
+[] ata1.00: Enabling discard_zeroes_data
+[] sd 0:0:0:0: [sda] Attached SCSI disk
+
+2. To locate a raw partition on the disk, run:
+  
+  $ sudo blkid
+  
+/dev/sdb1: UUID="ffffffff-ffff-ffff-ffff-ffffffffffff" TYPE="ext4" PARTUUID="ffffffff-ffff-ffff-ffff-ffffffffffff"
+/dev/sdb2: PARTUUID="ffffffff-ffff-ffff-ffff-ffffffffffff"
+
+* raw partition will have PARTUUID only 
+
+
+examples:\n
+
+I.
+* SSD is connected to PC using a docking station/enclosure/etc.
+* The data is on the raw partition /dev/sdb2
+* Destination path: /data/footage/test
+  Download:
+  * Whole partition in 10GB chunks:
+    {1}${0} -p /dev/sdb2 /data/footage/test{2}
+  * Whole partition in 1GB chunks:
+    {1}${0} -p /dev/sdb2 -bc 50 /data/footage/test{2}
+  * 5GB in 1GB chunks, skip the first 3GBs:
+    {1}${0} -p /dev/sdb2 -bc 50 -s 3 -n 5 /data/footage/test{2}
+
+Note: Repeat for any other external SSDs of the multi-camera system
+
+""".format(sys.argv[0],bcolors.OKGREEN,bcolors.ENDC)
+
+parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter)
+parser.add_argument("-p","--partition",default="",help="partition name, example: /dev/sdb2")
+parser.add_argument("-s","--skip",type=int,default=0, help="Number of chunks to skip from the beginning")
+parser.add_argument("-n","--n",type=int,default=0,help="Number of chunks to download")
+parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
+parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
+parser.add_argument("dest",help="desitnation directory: /data/footage/test")
+
+args = parser.parse_args()
+
+if args.partition!="":
+  pc = x393.PC()
+  if args.n==0:
+    args.bc = 512*1000
+    args.n = 1
+  pc.download(args.dest,args.partition,args.bs,args.bc,args.skip,args.n)
+else:
+  print(bcolors.WARNING+"Please specify a partition"+bcolors.ENDC)
+  print(usage)
+

int_ssd_dl_x393.py

+#!/usr/bin/env python
+
+# Download footage from ssd's raw partition:
+# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off
+#   ** All information is in the Exif headers, so no need to match SSDs to subcameras
+# * camera's internal SSD, raw partition, camera is powered on.
+#
+
+__author__ = "Elphel"
+__copyright__ = "Copyright 2016, Elphel, Inc."
+__license__ = "GPL"
+__version__ = "3.0+"
+__maintainer__ = "Oleg K Dzhimiev"
+__email__ = "oleg@elphel.com"
+__status__ = "Development"
+
+import argparse
+import os
+import sys
+import time
+
+import x393
+from x393 import bcolors
+
+# TODO: Download imu logs
+
+ssd_detect_timeout = 20
+
+usage = """
+
+keys:
+  To access camera without a password run:
+  
+  $ ssh-copy-id root@192.168.0.9
+  
+  To create keys, run:
+  
+  $ ssh-keygen
+
+examples:\n
+
+I.
+* internal SSD, camera is powered on
+* The data is on the raw partition /dev/sdb2
+* Destination path: /data/footage/test
+  Download:
+  * Whole partition in 10GB chunks:
+    {1}${0} -c root@192.168.0.9 -p /dev/sdb2 /data/footage/test{2}
+  * Whole partition in 1GB chunks:
+    {1}${0} -c root@192.168.0.9 -p /dev/sdb2 -bc 50 /data/footage/test{2}
+  * 5GB in 1GB chunks, skip the first 3GBs:
+    {1}${0} -c root@192.168.0.9 -p /dev/sdb2 -bc 50 -s 3 -n 5 /data/footage/test{2}
+    
+II. multiple cameras modes:
+  * "eyesis4pi"
+      * Whole partition in 10GB chunks:
+        {1}${0} -m eyesis4pi /data/footage/test{2}  
+
+""".format(sys.argv[0],bcolors.OKGREEN,bcolors.ENDC)
+
+parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter)
+parser.add_argument("-c","--camera", default="", help="provide camera ip address if downloading data from internal SSD, default = root@192.168.0.9")
+parser.add_argument("-m","--mode",default="",help="preset for multiple cameras, available modes: 'eyesis4pi'")
+parser.add_argument("-s","--skip",type=int,default=0, help="Number of chunks to skip from the beginning")
+parser.add_argument("-n","--n",type=int,default=0,help="Number of chunks to download")
+parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
+parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
+parser.add_argument("dest",help="desitnation directory: /data/footage/test")
+
+args = parser.parse_args()
+
+cams = []
+dirs = []
+
+if args.camera!="":
+  tmp = args.camera.split("@")
+  if len(tmp)==2:
+    cams = [{"user":tmp[0],"ip":tmp[1],"imu_log":0}]
+elif args.mode=="eyesis4pi":
+  cams = [
+    {"user":"root","ip":"192.168.0.161","imu_log":0},
+    {"user":"root","ip":"192.168.0.162","imu_log":0},
+    {"user":"root","ip":"192.168.0.163","imu_log":0}
+    ]
+else:
+  cams = [{"user":"root","ip":"192.168.0.9","imu_log":0}]
+  print("running default: "+cams[0]['user']+"@"+cams[0]['ip'])
+
+# init checks all connections
+for cam in cams:
+  cam['obj'] = x393.Camera(cam['user'],cam['ip'])
+  if cam['obj'].disable:
+    print(bcolors.WARNING+"Will skip "+cam['user']+"@"+cam['ip']+bcolors.ENDC)
+
+cams[:] = [tmp for tmp in cams if not tmp['obj'].disable]
+
+if len(cams)==0:
+  print(usage)
+
+pc = x393.PC()
+
+# get raw partitions
+for cam in cams:
+  d = cam['obj'].first_found_raw_partition_name()
+  if d!="undefined":
+    dirs.append(d)
+    print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)
+  else:
+    cam['disable']=1
+    print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)
+    #raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")
+
+# no need
+cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]
+
+# switch ssd to pc for all (not caring which cable is in)
+for cam in cams:
+  cam['obj'].ssd_to_pc()
+
+# download
+
+plist = []
+all_downloaded = False
+
+for i in range(len(cams)):
+  raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)
+  t = 0
+  while not all_downloaded:
+    plist = pc.list_partitions()
+    for d in dirs:
+      for p in plist:
+        if d==p[0]:
+          pc.download(args.dest,"/dev/"+p[1],d,args.bs,args.bc,args.skip,args.n)
+          dirs.delete(d)
+          if len(dirs)!=0:
+            print("wait for the next ssd")
+          else:
+            all_downloaded = True
+          break
+    print("Waiting for disks to show up:")
+    print(dirs)
+    t = t + 1
+    time.sleep(1)
+    if t>ssd_detect_timeout:
+      print(bcolors.FAIL+"PC: Error: Disks were not found:")
+      print(dirs)
+      print(bcolors.ENDC)
+      break
+    #cam['rp_name'].ssd_to_pc()
+
+# ssd to camera for all
+for cam in cams:
+  cam['obj'].ssd_to_camera()
+
+print("Done")
\ No newline at end of file

x393.py

+__author__ = "Elphel"
+__copyright__ = "Copyright 2016, Elphel, Inc."
+__license__ = "GPL"
+__version__ = "3.0+"
+__maintainer__ = "Oleg K Dzhimiev"
+__email__ = "oleg@elphel.com"
+__status__ = "Development"
+
+import os
+import re
+import subprocess
+import time
+
+def shout(cmd):
+  #subprocess.call prints to console
+  #subprocess.call(cmd,shell=True)
+  ret_str = ""
+  try:
+    ret_str = subprocess.check_output(cmd,shell=True)
+  except subprocess.CalledProcessError as e:
+    ret_str = str(e.returncode)
+  return ret_str
+
+#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[38;5;214m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    BOLDWHITE = '\033[1;37m'
+    UNDERLINE = '\033[4m'
+
+class Camera:
+  def __init__(self,user="root",ip="192.168.0.9"):
+    self.user = user
+    self.ip = ip
+    self.sshcmd = "ssh "+user+"@"+ip
+    self.disable = False
+    self.pattern = "ata-"
+    self.check_connection()
+    
+  def check_connection(self):
+    check = shout(self.sshcmd+" 'dmesg'")
+    if check.strip() == str(255):
+      self.disable = True
+    else:
+      print(self.user+"@"+self.ip+": connection ok")
+  
+  def ssd_to_pc(self):
+    self.unmount_all()
+    self.unload_ahci_driver()
+    self.mux_ssd_to_esata()
+    print(self.user+"@"+self.ip+": Enabled connection: internal SSD <-> PC")
+  
+  def ssd_to_camera(self):
+    self.load_ahci_driver()
+    self.mux_ssd_to_cpu()
+    print(self.user+"@"+self.ip+": Enabled connection: internal SSD <-> Camera")
+  
+  def partition_name(self,partition=""):
+    if partition!="":
+      res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id | grep '"+self.pattern+"' | grep '"+partition[-4:]+"''")
+      if res!="1":
+        for name in res.split(" "):
+          if name[0:len(self.pattern)]==self.pattern:
+            return name[len(self.pattern):]
+    return "undefined"
+  
+  def list_partitions(self):
+    res = shout(self.sshcmd+" 'cat /proc/partitions'")
+    plist = []
+    for line in res.splitlines():
+      s0 = re.search('sd[a-z][0-9]',line)
+      if s0:
+        plist.append(s0.group(0))
+      else:
+        s0 = re.search('sd[a-z]',line)
+        if s0:
+          plist.append(s0.group(0))
+    return plist
+  
+  def list_mounted_partitions(self):
+    res = shout(self.sshcmd+" 'df -h'")
+    df_list = []
+    for line in res.splitlines():
+      s0 = re.search('sd[a-z][0-9]',line)
+      if s0:
+        df_list.append(s0.group(0))
+    return df_list
+  
+  # combine with list_partitions?
+  def list_blkid(self):
+    res = shout(self.sshcmd+" 'blkid'")
+    blist = []
+    for line in res.splitlines():
+      s0 = re.search('sd[a-z][0-9]',line)
+      if s0:
+        s1 = re.search('TYPE=',line)
+        if s1:
+          blist.append(s0.group(0))
+    return blist
+  
+  def list_raw_partitions(self):
+    plist = self.list_partitions()
+    blist = self.list_blkid()
+        
+    for p in plist:
+      if p in blist:
+        plist.remove(p)
+        plist.remove(p[0:-1])
+    
+    return plist
+  
+  def first_found_raw_partition(self):
+    plist = self.list_raw_partitions()
+    if len(plist)!=0:
+      return "/dev/"+plist[0]
+    else:
+      return ""
+  
+  def first_found_raw_partition_name(self):
+    p = self.first_found_raw_partition()
+    return self.partition_name(p)
+  
+  def mux_ssd_to_esata(self):
+    shout(self.sshcmd+" '/usr/local/bin/x393sata_control.py set_esata_ssd'")
+
+  def mux_ssd_to_cpu(self):
+    shout(self.sshcmd+" '/usr/local/bin/x393sata_control.py set_zynq_ssd'")
+
+  def device_partitions(self,device):
+    test = shout(self.sshcmd+" 'ls /dev/disk/by-id/ -all'")
+    return test
+
+  def unmount_all(self):
+    blist = self.list_mounted_partitions()
+    if len(blist)==0:
+      print("umount: ok")
+    else:
+      for b in blist:
+        print("umounting /dev/"+b)
+        self.unmount("/dev/"+b)
+      
+  def unmount(self,partition):
+    shout(self.sshcmd+" 'umount "+partition+"'")
+
+  def unload_ahci_driver(self):
+    shout(self.sshcmd+" 'rmmod ahci_elphel'")
+  
+  def load_ahci_driver(self):
+    subprocess.Popen(self.sshcmd+" 'modprobe ahci_elphel'",shell=True)
+    time.sleep(1)
+    shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")
+
+class PC():
+  def __init__(self):
+    self.pattern = "ata-"
+  
+  def list_partitions(self):
+    res = shout("ls /dev/disk/by-id/ -all | grep '"+self.pattern+"'")
+    plist = []
+    for line in res.splitlines():
+      items = line.split(" ")
+      for name in items:
+        if name[0:len(self.pattern)]==self.pattern:
+          item = items[-1].split("/")
+          plist.append([name[len(self.pattern):],item[-1]])
+    return plist
+
+  def is_raw(self,part):
+    res = shout("sudo blkid | grep "+str(part))
+    typ = "TYPE="
+    s0 = re.search(typ,res)
+    if s0:
+      fstype = ""
+      pars = res.split(" ")
+      for par in pars:
+        if par[0:len(typ)]==typ:
+          fstype = par[len(typ):]
+          break
+      
+      print(part+" partition has a filesystem: "+fstype)
+      
+      raise Exception(bcolors.FAIL+"Partition has a filesystem ("+fstype+"): mount and copy files manually"+bcolors.ENDC)
+
+  def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
+
+    self.is_raw(part)
+
+    print("Getting raw partition data from "+part)
+    
+    if not os.path.isdir(dest):
+      os.mkdir(dest)
+    
+    dirname = self.partname(part)
+    
+    if dirname!="":
+      dirname = dest+"/"+dirname
+      if not os.path.isdir(dirname):
+        os.mkdir(dirname)
+      for i in range(0,dl_n+dl_skip):
+        fname = dirname+"/"+"file_"+str(i)+".img"
+        skip = i*(dl_bc-1)
+        if i>=dl_skip:
+          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))
+      
+    
+  def partname(self,partition):
+    cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"
+    res = shout(cmd)
+    for name in res.split(" "):
+      if name[0:len(self.pattern)]==self.pattern:
+        return name[len(self.pattern):]
+    return "" 
+