Commit ff5ea51b authored by Andrey Filippov's avatar Andrey Filippov

bug fixes

parent 58106de9
...@@ -19,6 +19,9 @@ ...@@ -19,6 +19,9 @@
* *
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
Found problem - does not process files that start in one chunk and end in another?
*/ */
define('START_TIFF', hex2bin('4d4d002a')); // start of a TIFF, JP/JP4 also have this TIFF marker define('START_TIFF', hex2bin('4d4d002a')); // start of a TIFF, JP/JP4 also have this TIFF marker
define('START_JP', hex2bin('ffd8ffe1')); // start of JPEG/JP4, TIFF usually does not have JPEG/JP4 markers define('START_JP', hex2bin('ffd8ffe1')); // start of JPEG/JP4, TIFF usually does not have JPEG/JP4 markers
...@@ -184,7 +187,9 @@ function split_file($path,$file,$destination,$add_to_chn=-1){ ...@@ -184,7 +187,9 @@ function split_file($path,$file,$destination,$add_to_chn=-1){
while(true){ while(true){
$pos=strpos($s,$startMarkerWithExif,$pos); $pos=strpos($s,$startMarkerWithExif,$pos);
if ($pos === false) break; if ($pos === false) break;
// echo count($markers).': '.strval(($offset+$pos)% 4096) .' ('.strval($offset+$pos).')'; // debugging
$markers[count($markers)]=$offset+$pos; $markers[count($markers)]=$offset+$pos;
$pos++; $pos++;
} }
$offset+=(strlen($s)-strlen($startMarkerWithExif)+1); // so each marker will appear once $offset+=(strlen($s)-strlen($startMarkerWithExif)+1); // so each marker will appear once
......
...@@ -65,13 +65,16 @@ II. multiple cameras modes: ...@@ -65,13 +65,16 @@ II. multiple cameras modes:
parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter) parser = argparse.ArgumentParser(description=usage,formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("-c","--camera", default="", help="provide camera ip address if downloading data from internal SSD, default = root@192.168.0.9") parser.add_argument("-c","--camera", default="", help="provide camera ip address if downloading data from internal SSD, default = root@192.168.0.9")
parser.add_argument("-m","--mode",default="",help="preset for multiple cameras, available modes: 'eyesis4pi'") parser.add_argument("-m","--mode",default="",help="preset for multiple cameras, available modes: 'eyesis4pi'")
parser.add_argument("-s","--skip",type=int,default=0, help="Number of chunks to skip from the beginning") parser.add_argument("-s","--skip",type=int,default=0, help="Number of blocks to skip from the beginning") # chunks to skip from the beginning")
parser.add_argument("-n","--n",type=int,default=0,help="Number of chunks to download") parser.add_argument("-n","--n",type=int,default=0,help="Number of blocks to download) # chunks to download")
parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20") parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB") parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
parser.add_argument("-fs","--file_start",default="",help=".disk file name with start LBA pointer, will calculate 'skip' value")
parser.add_argument("-fe","--file_end",default="camogm.disk",help=".disk file name with end LBA pointer, default is 'camogm.disk'")
parser.add_argument("dest",help="desitnation directory: /data/footage/test") parser.add_argument("dest",help="desitnation directory: /data/footage/test")
args = parser.parse_args() args = parser.parse_args()
print (args)
cams = [] cams = []
dirs = [] dirs = []
...@@ -141,8 +144,30 @@ for i in range(len(cams)): ...@@ -141,8 +144,30 @@ for i in range(len(cams)):
if d==p[0]: if d==p[0]:
# p[1] == sdb2 # p[1] == sdb2
# hardcoded /dev/sd?1 # hardcoded /dev/sd?1
if args.n==0:
data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
data_size_blocks -= data_skip_blocks
else:
data_size_blocks = args.n
data_skip_blocks = args.skip
if (data_size_blocks < 0):
data_size_blocks = 0 # if camogm.disk could not be read?
print ("data_size_blocks=%d, data_skip_blocks=%d"%(data_size_blocks,data_skip_blocks))
chunk_blocks=32768 # 4096
block_size= 512 # 4096
data_size_gb = (data_size_blocks * block_size) / (1024 * 1024 * 1024)
file_gb = args.bs*args.bc // 1024 # just for compatibility, change to parameter
print("Data size: %d %d-byte blocks (%f GB)"%(data_size_blocks, block_size, data_size_gb))
pc.download_blocks(args.dest, # def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512):
"/dev/"+p[1],
blocks_load=data_size_blocks,
blocks_skip= data_skip_blocks,
file_gb=file_gb,
chunk_blocks=chunk_blocks,
block_size=block_size)
'''
data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1") data_size = pc.read_camogm_disk_file("/dev/"+p[1][0:-1]+"1")
data_size = round(data_size,2) data_size = round(data_size,2)
# bs is in kB # bs is in kB
chunk_size = float(args.bs*args.bc)/1024 chunk_size = float(args.bs*args.bc)/1024
...@@ -153,8 +178,11 @@ for i in range(len(cams)): ...@@ -153,8 +178,11 @@ for i in range(len(cams)):
print("Data size: "+str(data_size)+" GB") print("Data size: "+str(data_size)+" GB")
print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC) print(bcolors.BOLDWHITE+"Download size: "+str(args.n)+"x "+str(round(chunk_size,2))+"GB, skipped the first "+str(args.skip)+" chunks"+bcolors.ENDC)
pc.download (args.dest, "/dev/"+p[1], args.bs,args.bc,args.skip,args.n)
pc.download(args.dest,"/dev/"+p[1],args.bs,args.bc,args.skip,args.n) '''
# def download(self,dest,part,dl_bs=20,dl_bc=512,dl_skip=0,dl_n=0):
#parser.add_argument("-bs",type=int,default=20,help="block size in MB, default = 20")
#parser.add_argument("-bc",type=int,default=512,help="Number of blocks of size [bs] in a single chunk, default = 512, so the default chunk size is 10GB")
dirs.remove(d) dirs.remove(d)
proceed_to_next = True proceed_to_next = True
if len(dirs)!=0: if len(dirs)!=0:
......
...@@ -173,7 +173,7 @@ class PC(): ...@@ -173,7 +173,7 @@ class PC():
# mounts partition (/dev/sd?1), reads camogm.disk file # mounts partition (/dev/sd?1), reads camogm.disk file
# returns the download size from raw partition ((/dev/sd?2)) # returns the download size from raw partition ((/dev/sd?2))
def read_camogm_disk_file(self,part): def read_camogm_disk_file(self,part,fname="camogm.disk"):
result = 0 result = 0
...@@ -182,16 +182,45 @@ class PC(): ...@@ -182,16 +182,45 @@ class PC():
shout("sudo mount "+part+" "+tmp_mount_point) shout("sudo mount "+part+" "+tmp_mount_point)
try: try:
with open (tmp_mount_point+"/camogm.disk", "r") as myfile: with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines() data=myfile.readlines()
if len(data)==2: if len(data)==2:
l2 = data[1] l2 = data[1]
pointers = l2.split("\t") pointers = l2.split("\t")
pntr1 = int(pointers[1]) pntr1 = int(pointers[1])
pntr2 = int(pointers[2]) pntr2 = int(pointers[2])
result = float(pntr2-pntr1)*512/1024/1024/1024 result = float(pntr2-pntr1)*512/1024/1024/1024 # still 512 block size
# result = float(pntr2-pntr1)*4096/1024/1024/1024 # blocks are now 4096, not 512 bytes!
except IOError: except IOError:
print(tmp_mount_point+"/camogm.disk NOT FOUND") print(tmp_mount_point+"/"+fname+" NOT FOUND")
shout("sudo umount "+tmp_mount_point)
os.rmdir(tmp_mount_point)
return result
"""
Returns block offset from the start of partition to the current LBA pointer
"""
def read_camogm_disk_file_blocks(self,part,fname="camogm.disk"):
result = 0
tmp_mount_point = tempfile.mkdtemp()
print("mounting "+part+" to "+tmp_mount_point)
shout("sudo mount "+part+" "+tmp_mount_point)
try:
with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines()
if len(data)==2:
l2 = data[1]
pointers = l2.split("\t")
pntr1 = int(pointers[1])
pntr2 = int(pointers[2])
# result = float(pntr2-pntr1)*512/1024/1024/1024
result = pntr2-pntr1
except IOError:
print(tmp_mount_point+"/"+fname+" NOT FOUND")
shout("sudo umount "+tmp_mount_point) shout("sudo umount "+tmp_mount_point)
os.rmdir(tmp_mount_point) os.rmdir(tmp_mount_point)
...@@ -236,6 +265,50 @@ class PC(): ...@@ -236,6 +265,50 @@ class PC():
if i>=dl_skip: if i>=dl_skip:
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip)) shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))
# def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=4096, block_size=4096):
def download_blocks(self, dest, part, blocks_load, blocks_skip= 0, file_gb=10, chunk_blocks=32768, block_size=512): #4096):
chunk_bytes = block_size * chunk_blocks
file_chunks = (file_gb * 1024 * 1024 * 1024) // chunk_bytes
self.is_raw(part)
print("Getting raw partition data from "+part)
if not os.path.isdir(dest):
os.mkdir(dest)
dirname = self.partname(part)
if dirname!="":
dirname = dest+"/"+dirname
if not os.path.isdir(dirname):
os.mkdir(dirname)
num_file = 0
# optional first file to align skip to chunk_blocks, 1 block at a time
if (blocks_skip > 0) and ((blocks_skip // chunk_blocks) > 0):
bwrite = chunk_blocks - (blocks_skip // chunk_blocks)
if (bwrite > blocks_load):
bwrite = blocks_load
fname = dirname+"/"+"file_"+str(num_file)+".img"
print("Aligning skip to chunks, writing %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
blocks_skip += bwrite
blocks_load -= bwrite
num_file += 1
# write bulk of the blocks, <= file_chunks of chunks in each file
while ((blocks_load // chunk_blocks) > 0):
chunks_write = blocks_load // chunk_blocks
chunks_skip = blocks_skip // chunk_blocks # should be already multiple of chunks
if (chunks_write > file_chunks):
chunks_write = file_chunks
fname = dirname+"/"+"file_"+str(num_file)+".img"
print("Writing %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
bwrite = chunks_write * chunk_blocks
blocks_skip += bwrite
blocks_load -= bwrite
num_file += 1
# optionally write the remainder (< chunk), 1 block at a time
if (blocks_load > 0):
fname = dirname+"/"+"file_"+str(num_file)+".img"
print("Writing last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
def partname(self,partition): def partname(self,partition):
cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'" cmd = "ls /dev/disk/by-id/ -all | grep '"+self.pattern+"' | grep '"+partition[-4:]+"'"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment