initial

#!/usr/bin/env python

# Download footage from ssd's raw partition:

# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off

#   ** All information is in the Exif headers, so no need to match SSDs to subcameras

# * camera's internal SSD, raw partition, camera is powered on.

#

__author__ = "Elphel"

__copyright__ = "Copyright 2016, Elphel, Inc."

__license__ = "GPL"

__version__ = "3.0+"

__maintainer__ = "Oleg K Dzhimiev"

__email__ = "oleg@elphel.com"

__status__ = "Development"

import argparse

import os

import sys

import time

import x393

from x393 import bcolors

usage = """

help:

1. To locate a newly connected SSD device, run:

  $ dmesg

[] scsi 0:0:0:0: Direct-Access     ATA      Crucial_CT250MX2 MU02 PQ: 0 ANSI: 5

[] ata1.00: Enabling discard_zeroes_data

[] sd 0:0:0:0: Attached scsi generic sg0 type 0

[] sd 0:0:0:0: [sdb] 488397168 512-byte logical blocks: (250 GB/232 GiB)

[] sd 0:0:0:0: [sdb] 4096-byte physical blocks

[] sd 0:0:0:0: [sdb] Write Protect is off

[] sd 0:0:0:0: [sdb] Mode Sense: 00 3a 00 00

[] sd 0:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[] ata1.00: Enabling discard_zeroes_data

[]  sdb: sdb1 sdb2

[] ata1.00: Enabling discard_zeroes_data

[] sd 0:0:0:0: [sda] Attached SCSI disk

2. To locate a raw partition on the disk, run:

  $ sudo blkid

/dev/sdb1: UUID="ffffffff-ffff-ffff-ffff-ffffffffffff" TYPE="ext4" PARTUUID="ffffffff-ffff-ffff-ffff-ffffffffffff"

/dev/sdb2: PARTUUID="ffffffff-ffff-ffff-ffff-ffffffffffff"

* raw partition will have PARTUUID only 

examples:\n

I.

* SSD is connected to PC using a docking station/enclosure/etc.

* The data is on the raw partition /dev/sdb2

* Destination path: /data/footage/test

  Download:

  * Whole partition in 10GB chunks:

    {1}${0} -p /dev/sdb2 /data/footage/test{2}

  * Whole partition in 1GB chunks:

    {1}${0} -p /dev/sdb2 -bc 50 /data/footage/test{2}

  * 5GB in 1GB chunks, skip the first 3GBs:

    {1}${0} -p /dev/sdb2 -bc 50 -s 3 -n 5 /data/footage/test{2}

Note: Repeat for any other external SSDs of the multi-camera system

""".format(sys.argv[0],bcolors.OKGREEN,bcolors.ENDC)

parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter)

parser.add_argument("-p","--partition",default="",help="partition name, example: /dev/sdb2")

parser.add_argument("-s","--skip",type=int,default=0, help="Number of chunks to skip from the beginning")

parser.add_argument("-n","--n",type=int,default=0,help="Number of chunks to download")

parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")

parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")

parser.add_argument("dest",help="desitnation directory: /data/footage/test")

args = parser.parse_args()

if args.partition!="":

  pc = x393.PC()

  if args.n==0:

    args.bc = 512*1000

    args.n = 1

  pc.download(args.dest,args.partition,args.bs,args.bc,args.skip,args.n)

else:

  print(bcolors.WARNING+"Please specify a partition"+bcolors.ENDC)

  print(usage)

#!/usr/bin/env python

# Download footage from ssd's raw partition:

# * standalone SSDs connected using a docking station or enclosure or else, raw partition, camera can be powered off

#   ** All information is in the Exif headers, so no need to match SSDs to subcameras

# * camera's internal SSD, raw partition, camera is powered on.

#

__author__ = "Elphel"

__copyright__ = "Copyright 2016, Elphel, Inc."

__license__ = "GPL"

__version__ = "3.0+"

__maintainer__ = "Oleg K Dzhimiev"

__email__ = "oleg@elphel.com"

__status__ = "Development"

import argparse

import os

import sys

import time

import x393

from x393 import bcolors

# TODO: Download imu logs

ssd_detect_timeout = 20

usage = """

keys:

  To access camera without a password run:

  $ ssh-copy-id root@192.168.0.9

  To create keys, run:

  $ ssh-keygen

examples:\n

I.

* internal SSD, camera is powered on

* The data is on the raw partition /dev/sdb2

* Destination path: /data/footage/test

  Download:

  * Whole partition in 10GB chunks:

    {1}${0} -c root@192.168.0.9 -p /dev/sdb2 /data/footage/test{2}

  * Whole partition in 1GB chunks:

    {1}${0} -c root@192.168.0.9 -p /dev/sdb2 -bc 50 /data/footage/test{2}

  * 5GB in 1GB chunks, skip the first 3GBs:

    {1}${0} -c root@192.168.0.9 -p /dev/sdb2 -bc 50 -s 3 -n 5 /data/footage/test{2}

II. multiple cameras modes:

  * "eyesis4pi"

      * Whole partition in 10GB chunks:

        {1}${0} -m eyesis4pi /data/footage/test{2}  

""".format(sys.argv[0],bcolors.OKGREEN,bcolors.ENDC)

parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter)

parser.add_argument("-c","--camera", default="", help="provide camera ip address if downloading data from internal SSD, default = root@192.168.0.9")

parser.add_argument("-m","--mode",default="",help="preset for multiple cameras, available modes: 'eyesis4pi'")

parser.add_argument("-s","--skip",type=int,default=0, help="Number of chunks to skip from the beginning")

parser.add_argument("-n","--n",type=int,default=0,help="Number of chunks to download")

parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")

parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")

parser.add_argument("dest",help="desitnation directory: /data/footage/test")

args = parser.parse_args()

cams = []

dirs = []

if args.camera!="":

  tmp = args.camera.split("@")

  if len(tmp)==2:

    cams = [{"user":tmp[0],"ip":tmp[1],"imu_log":0}]

elif args.mode=="eyesis4pi":

  cams = [

    {"user":"root","ip":"192.168.0.161","imu_log":0},

    {"user":"root","ip":"192.168.0.162","imu_log":0},

    {"user":"root","ip":"192.168.0.163","imu_log":0}

    ]

else:

  cams = [{"user":"root","ip":"192.168.0.9","imu_log":0}]

  print("running default: "+cams[0]['user']+"@"+cams[0]['ip'])

# init checks all connections

for cam in cams:

  cam['obj'] = x393.Camera(cam['user'],cam['ip'])

  if cam['obj'].disable:

    print(bcolors.WARNING+"Will skip "+cam['user']+"@"+cam['ip']+bcolors.ENDC)

cams[:] = [tmp for tmp in cams if not tmp['obj'].disable]

if len(cams)==0:

  print(usage)

pc = x393.PC()

# get raw partitions

for cam in cams:

  d = cam['obj'].first_found_raw_partition_name()

  if d!="undefined":

    dirs.append(d)

    print(cam['user']+"@"+cam['ip']+": raw partition name: "+d)

  else:

    cam['disable']=1

    print(bcolors.FAIL+cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found"+bcolors.ENDC)

    #raise Exception(cam['user']+"@"+cam['ip']+" : error: already switched or raw partition not found")

# no need

cams[:] = [tmp for tmp in cams if tmp.get('disable')!=1]

# switch ssd to pc for all (not caring which cable is in)

for cam in cams:

  cam['obj'].ssd_to_pc()

# download

plist = []

all_downloaded = False

for i in range(len(cams)):

  raw_input(bcolors.OKGREEN+"Connect camera (eSATA) to PC (eSATA/SATA). Press Enter to continue..."+bcolors.ENDC)

  t = 0

  while not all_downloaded:

    plist = pc.list_partitions()

    for d in dirs:

      for p in plist:

        if d==p[0]:

          pc.download(args.dest,"/dev/"+p[1],d,args.bs,args.bc,args.skip,args.n)

          dirs.delete(d)

          if len(dirs)!=0:

            print("wait for the next ssd")

          else:

            all_downloaded = True

          break

    print("Waiting for disks to show up:")

    print(dirs)

    t = t + 1

    time.sleep(1)

    if t>ssd_detect_timeout:

      print(bcolors.FAIL+"PC: Error: Disks were not found:")

      print(dirs)

      print(bcolors.ENDC)

      break

    #cam['rp_name'].ssd_to_pc()

# ssd to camera for all

for cam in cams:

  cam['obj'].ssd_to_camera()

print("Done")

 No newline at end of file

__author__ = "Elphel"

__copyright__ = "Copyright 2016, Elphel, Inc."

__license__ = "GPL"

__version__ = "3.0+"

__maintainer__ = "Oleg K Dzhimiev"

__email__ = "oleg@elphel.com"

__status__ = "Development"

import os

import re

import subprocess

import time

def shout(cmd):

  #subprocess.call prints to console

  #subprocess.call(cmd,shell=True)

  ret_str = ""

  try:

    ret_str = subprocess.check_output(cmd,shell=True)

  except subprocess.CalledProcessError as e:

    ret_str = str(e.returncode)

  return ret_str

#http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python

class bcolors:

    HEADER = '\033[95m'

    OKBLUE = '\033[94m'

    OKGREEN = '\033[92m'

    WARNING = '\033[38;5;214m'

    FAIL = '\033[91m'

    ENDC = '\033[0m'

    BOLD = '\033[1m'

    BOLDWHITE = '\033[1;37m'

    UNDERLINE = '\033[4m'

class Camera:

  def __init__(self,user="root",ip="192.168.0.9"):

    self.user = user

    self.ip = ip

    self.sshcmd = "ssh "+user+"@"+ip

    self.disable = False

    self.pattern = "ata-"

    self.check_connection()

  def check_connection(self):

    check = shout(self.sshcmd+" 'dmesg'")

    if check.strip() == str(255):

      self.disable = True

    else:

      print(self.user+"@"+self.ip+": connection ok")

  def ssd_to_pc(self):

    self.unmount_all()

    self.unload_ahci_driver()

    self.mux_ssd_to_esata()

    print(self.user+"@"+self.ip+": Enabled connection: internal SSD <-> PC")

  def ssd_to_camera(self):

    self.load_ahci_driver()

    self.mux_ssd_to_cpu()

    print(self.user+"@"+self.ip+": Enabled connection: internal SSD <-> Camera")

  def partition_name(self,partition=""):

    if partition!="":

      res = shout(self.sshcmd+" 'ls -all /dev/disk/by-id | grep '"+self.pattern+"' | grep '"+partition[-4:]+"''")

      if res!="1":

        for name in res.split(" "):

          if name[0:len(self.pattern)]==self.pattern:

            return name[len(self.pattern):]

    return "undefined"

  def list_partitions(self):

    res = shout(self.sshcmd+" 'cat /proc/partitions'")

    plist = []

    for line in res.splitlines():

      s0 = re.search('sd[a-z][0-9]',line)

      if s0:

        plist.append(s0.group(0))

      else:

        s0 = re.search('sd[a-z]',line)

        if s0:

          plist.append(s0.group(0))

    return plist

  def list_mounted_partitions(self):

    res = shout(self.sshcmd+" 'df -h'")

    df_list = []

    for line in res.splitlines():

      s0 = re.search('sd[a-z][0-9]',line)

      if s0:

        df_list.append(s0.group(0))

    return df_list

  # combine with list_partitions?

  def list_blkid(self):

    res = shout(self.sshcmd+" 'blkid'")

    blist = []

    for line in res.splitlines():

      s0 = re.search('sd[a-z][0-9]',line)

      if s0:

        s1 = re.search('TYPE=',line)

        if s1:

          blist.append(s0.group(0))

    return blist

  def list_raw_partitions(self):

    plist = self.list_partitions()

    blist = self.list_blkid()

    for p in plist:

      if p in blist:

        plist.remove(p)

        plist.remove(p[0:-1])

    return plist

  def first_found_raw_partition(self):

    plist = self.list_raw_partitions()

    if len(plist)!=0:

      return "/dev/"+plist[0]

    else:

      return ""

  def first_found_raw_partition_name(self):

    p = self.first_found_raw_partition()

    return self.partition_name(p)

  def mux_ssd_to_esata(self):

    shout(self.sshcmd+" '/usr/local/bin/x393sata_control.py set_esata_ssd'")

  def mux_ssd_to_cpu(self):

    shout(self.sshcmd+" '/usr/local/bin/x393sata_control.py set_zynq_ssd'")

  def device_partitions(self,device):

    test = shout(self.sshcmd+" 'ls /dev/disk/by-id/ -all'")

    return test

  def unmount_all(self):

    blist = self.list_mounted_partitions()

    if len(blist)==0:

      print("umount: ok")

    else:

      for b in blist:

        print("umounting /dev/"+b)

        self.unmount("/dev/"+b)

  def unmount(self,partition):

    shout(self.sshcmd+" 'umount "+partition+"'")

  def unload_ahci_driver(self):

    shout(self.sshcmd+" 'rmmod ahci_elphel'")

  def load_ahci_driver(self):

    subprocess.Popen(self.sshcmd+" 'modprobe ahci_elphel'",shell=True)

    time.sleep(1)

    shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")

class PC():

  def __init__(self):

    self.pattern = "ata-"

  def list_partitions(self):

    res = shout("ls /dev/disk/by-id/ -all | grep '"+self.pattern+"'")

    plist = []

    for line in res.splitlines():

      items = line.split(" ")

      for name in items:

        if name[0:len(self.pattern)]==self.pattern:

          item = items[-1].split("/")

          plist.append([name[len(self.pattern):],item[-1]])

    return plist

  def is_raw(self,part):

    res = shout("sudo blkid | grep "+str(part))

    typ = "TYPE="

    s0 = re.search(typ,res)

    if s0:

      fstype = ""

      pars = res.split(" ")

      for par in pars:

        if par[0:len(typ)]==typ:

          fstype = par[len(typ):]

          break

      print(part+" partition has a filesystem: "+fstype)

      raise Exception(bcolors.FAIL+"Partition has a filesystem ("+fstype+"): mount and copy files manually"+bcolors.ENDC)

  def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):

    self.is_raw(part)

    print("Getting raw partition data from "+part)

    if not os.path.isdir(dest):

      os.mkdir(dest)

    dirname = self.partname(part)

    if dirname!="":

      dirname = dest+"/"+dirname

      if not os.path.isdir(dirname):

        os.mkdir(dirname)

      for i in range(0,dl_n+dl_skip):

        fname = dirname+"/"+"file_"+str(i)+".img"

        skip = i*(dl_bc-1)

        if i>=dl_skip:

          shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))

  def partname(self,partition):

    cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"

    res = shout(cmd)

    for name in res.split(" "):

      if name[0:len(self.pattern)]==self.pattern:

        return name[len(self.pattern):]

    return ""