bug fixes

extract_images_tiff.php

@@ -19,6 +19,9 @@
  *
  *  You should have received a copy of the GNU General Public License
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/*
+Found problem - does not process files that start in one chunk and end in another?
 */
    define('START_TIFF',     hex2bin('4d4d002a')); // start of a TIFF, JP/JP4 also have this TIFF marker
    define('START_JP',       hex2bin('ffd8ffe1')); // start of JPEG/JP4, TIFF usually does not have JPEG/JP4 markers
@@ -184,7 +187,9 @@ function split_file($path,$file,$destination,$add_to_chn=-1){
       while(true){
         $pos=strpos($s,$startMarkerWithExif,$pos);
         if ($pos === false) break;
+//        echo count($markers).': '.strval(($offset+$pos)% 4096) .' ('.strval($offset+$pos).')'; // debugging
         $markers[count($markers)]=$offset+$pos;
+        
         $pos++;
       }
       $offset+=(strlen($s)-strlen($startMarkerWithExif)+1); // so each marker will appear once

int_ssd_download.py

@@ -65,13 +65,16 @@ II. multiple cameras modes:
 parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter)
 parser.add_argument("-c","--camera", default="", help="provide camera ip address if downloading data from internal SSD, default = root@192.168.0.9")
 parser.add_argument("-m","--mode",default="",help="preset for multiple cameras, available modes: 'eyesis4pi'")
-parser.add_argument("-s","--skip",type=int,default=0, help="Number of chunks to skip from the beginning")
-parser.add_argument("-n","--n",type=int,default=0,help="Number of chunks to download")
+parser.add_argument("-s","--skip",type=int,default=0, help="Number of blocks to skip from the beginning") # chunks to skip from the beginning")
+parser.add_argument("-n","--n",type=int,default=0,help="Number of blocks to download) # chunks to download") 
 parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
 parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
+parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")
+parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")
 parser.add_argument("dest",help="desitnation directory: /data/footage/test")
 
 args = parser.parse_args()
+print (args)
 
 cams = []
 dirs = []
@@ -141,8 +144,30 @@ for i in range(len(cams)):
         if d==p[0]:
           # p[1] == sdb2
           # hardcoded /dev/sd?1
+          if args.n==0:
+            data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
+            data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
+            data_size_blocks -= data_skip_blocks 
+          else:
+            data_size_blocks = args.n
+            data_skip_blocks = args.skip
+          if (data_size_blocks < 0):
+              data_size_blocks = 0 # if camogm.disk could not be read?  
+          print ("data_size_blocks=%d, data_skip_blocks=%d"%(data_size_blocks,data_skip_blocks))  
+          chunk_blocks=32768 # 4096
+          block_size=    512 # 4096
+          data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)
+          file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter
+          print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))  
+          pc.download_blocks(args.dest, #  def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512):
+                             "/dev/"+p[1], 
+                             blocks_load=data_size_blocks,
+                             blocks_skip= data_skip_blocks,
+                             file_gb=file_gb,
+                             chunk_blocks=chunk_blocks,
+                             block_size=block_size)
+          '''
           data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1")
-
           data_size = round(data_size,2)
           # bs is in kB
           chunk_size = float(args.bs*args.bc)/1024
@@ -153,8 +178,11 @@ for i in range(len(cams)):
 
           print("Data size: "+str(data_size)+" GB")
           print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC)
-
-          pc.download(args.dest,"/dev/"+p[1],args.bs,args.bc,args.skip,args.n)
+          pc.download       (args.dest, "/dev/"+p[1], args.bs,args.bc,args.skip,args.n)
+          '''
+#  def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
+#parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
+#parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
           dirs.remove(d)
           proceed_to_next = True
           if len(dirs)!=0:

x393.py

@@ -173,7 +173,7 @@ class PC():
 
   # mounts partition (/dev/sd?1), reads camogm.disk file
   # returns the download size from raw partition ((/dev/sd?2))
-  def read_camogm_disk_file(self,part):
+  def read_camogm_disk_file(self,part,fname="camogm.disk"):
 
     result = 0
 
@@ -182,16 +182,45 @@ class PC():
     shout("sudo mount "+part+" "+tmp_mount_point)
 
     try:
-      with open (tmp_mount_point+"/camogm.disk", "r") as myfile:
+      with open (tmp_mount_point+"/"+fname, "r") as myfile:
         data=myfile.readlines()
         if len(data)==2:
           l2 = data[1]
           pointers = l2.split("\t")
           pntr1 = int(pointers[1])
           pntr2 = int(pointers[2])
-          result = float(pntr2-pntr1)*512/1024/1024/1024
+          result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
+#          result = float(pntr2-pntr1)*4096/1024/1024/1024 # blocks are now 4096, not 512 bytes!
     except IOError:
-      print(tmp_mount_point+"/camogm.disk NOT FOUND")
+      print(tmp_mount_point+"/"+fname+" NOT FOUND")
+
+    shout("sudo umount "+tmp_mount_point)
+    os.rmdir(tmp_mount_point)
+
+    return result
+
+  """
+  Returns block offset from the start of partition to the current LBA pointer
+  """
+  def read_camogm_disk_file_blocks(self,part,fname="camogm.disk"):
+    result = 0
+    tmp_mount_point = tempfile.mkdtemp()
+    print("mounting "+part+" to "+tmp_mount_point)
+    shout("sudo mount "+part+" "+tmp_mount_point)
+
+    try:
+      with open (tmp_mount_point+"/"+fname, "r") as myfile:
+        data=myfile.readlines()
+        if len(data)==2:
+          l2 = data[1]
+          pointers = l2.split("\t")
+          pntr1 = int(pointers[1])
+          
+          pntr2 = int(pointers[2])
+#          result = float(pntr2-pntr1)*512/1024/1024/1024
+          result = pntr2-pntr1
+    except IOError:
+      print(tmp_mount_point+"/"+fname+" NOT FOUND")
 
     shout("sudo umount "+tmp_mount_point)
     os.rmdir(tmp_mount_point)
@@ -236,6 +265,50 @@ class PC():
         if i>=dl_skip:
           shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))
       
+#  def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=4096, block_size=4096):
+  def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
+    chunk_bytes =  block_size * chunk_blocks 
+    file_chunks =  (file_gb * 1024 * 1024 * 1024) // chunk_bytes
+    self.is_raw(part)
+    print("Getting raw partition data from "+part)
+    if not os.path.isdir(dest):
+      os.mkdir(dest)
+    dirname = self.partname(part)
+    if dirname!="":
+      dirname = dest+"/"+dirname
+      if not os.path.isdir(dirname):
+        os.mkdir(dirname)
+      num_file = 0
+      # optional first file to align skip to chunk_blocks, 1 block at a time
+      if (blocks_skip > 0) and ((blocks_skip // chunk_blocks) > 0):
+        bwrite = chunk_blocks - (blocks_skip // chunk_blocks)
+        if (bwrite > blocks_load):
+          bwrite = blocks_load
+        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        print("Aligning skip to chunks, writing %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
+        shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
+        blocks_skip += bwrite
+        blocks_load -= bwrite
+        num_file += 1
+      # write bulk of the blocks, <= file_chunks of chunks in each file
+      while ((blocks_load // chunk_blocks)  > 0):
+        chunks_write = blocks_load // chunk_blocks
+        chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
+        if (chunks_write > file_chunks):
+          chunks_write = file_chunks
+        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        print("Writing %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
+        shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
+        bwrite = chunks_write * chunk_blocks
+        blocks_skip += bwrite
+        blocks_load -= bwrite
+        num_file += 1
+      # optionally write the remainder (< chunk), 1 block at a time
+      if (blocks_load > 0):
+        fname = dirname+"/"+"file_"+str(num_file)+".img"
+        print("Writing last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
+        shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
+    
     
   def partname(self,partition):
     cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"