Skip to content

E
elphel-tools-x393
Commit d5901897 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

correctly processing split files, modified filenames for sorting

parent b46a9a1d
Show whitespace changes
Inline Side-by-side
Showing with 244 additions and 124 deletions
extract_images_tiff.php
View file @ d5901897
...@@ -21,12 +21,22 @@ ...@@ -21,12 +21,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
/* /*
Found problem - does not process files that start in one chunk and end in another? Found problem - does not process files that start in one file and end in another?
*/ */
define('START_TIFF', hex2bin('4d4d002a')); // start of a TIFF, JP/JP4 also have this TIFF marker define('START_TIFF', hex2bin('4d4d002a')); // start of a TIFF, JP/JP4 also have this TIFF marker
define('START_JP', hex2bin('ffd8ffe1')); // start of JPEG/JP4, TIFF usually does not have JPEG/JP4 markers define('START_JP', hex2bin('ffd8ffe1')); // start of JPEG/JP4, TIFF usually does not have JPEG/JP4 markers
define('START_EXIF', hex2bin("457869660000")); // Exif\0\0
define('EXIF_OFFSET', 6); // Exif start offset from the beginning of the file (from start of START_JP)
define('END_JP', hex2bin('ffd9')); // end of JPEG/JP4 file
define('MAX_HEAD_LEN', 0x300); // maximal header length that includes complete header define('MAX_HEAD_LEN', 0x300); // maximal header length that includes complete header
//disable the default time limit for php scripts. define('MIN_JP_LEN', 0x500); // skip from jpeg looking for end marker
define('MAX_IMG_LEN', 0x1000000); // maximal image size (16M) to look for the end marker
define('REQUIRE_EXIF', 1); // JPEG/JP4 must have Exif (more filtering, less false positives)
//define('DEBUG_LEVEL', 3); // Generate debug output
//define('DEBUG_OUT_FILES',DEBUG_LEVEL? 1 : 0 ); // 1 to add sequence number to result image names (make all unique)
define('DEBUG_LEVEL', 1); // Generate debug output
define('DEBUG_OUT_FILES',DEBUG_LEVEL? 0 : 0 ); // 1 to add sequence number to result image names (make all unique)
//disable the default time limit for php scripts.
set_time_limit(0); set_time_limit(0);
$chunksize=10000000; //10MB $chunksize=10000000; //10MB
...@@ -50,7 +60,7 @@ function print_help(){ ...@@ -50,7 +60,7 @@ function print_help(){
global $argv; global $argv;
echo <<<"TXT" echo <<<"TXT"
Help: Help:
* Usage: * Usage:
~$ {$argv[0]} path=[path-to-dir] dest_path=[dest-subdir] move_processed=[move-processed-files] ext=[forced-ext] chn_offs=[add-to-chn] ~$ {$argv[0]} path=[path-to-dir] dest_path=[dest-subdir] move_processed=[move-processed-files] ext=[forced-ext] chn_offs=[add-to-chn]
...@@ -69,7 +79,7 @@ Help: ...@@ -69,7 +79,7 @@ Help:
** Split all *.img, *.bin and *.mov files in the current dir and 1 dir down, puts results to 'results/', override extensions with 'jpg': ** Split all *.img, *.bin and *.mov files in the current dir and 1 dir down, puts results to 'results/', override extensions with 'jpg':
~$ {$argv[0]} dest_path=results ext=jpg ~$ {$argv[0]} dest_path=results ext=jpg
TXT; TXT;
} }
...@@ -115,25 +125,48 @@ if (isset($_GET['chn_offs'])){ ...@@ -115,25 +125,48 @@ if (isset($_GET['chn_offs'])){
} }
$list = preg_grep('/^([^.])/', scandir($path)); $list = preg_grep('/^([^.])/', scandir($path));
sort($list); // also re-indexes
//$list=array();
if (!is_dir("$path/$destination")) mkdir("$path/$destination",0777);
foreach ($list as $item) { if (!is_dir("$path/$destination")) mkdir("$path/$destination",0777);
if (is_dir("$path/$item")){ if ((DEBUG_LEVEL >= 2)) {
if ($item==$processed_subdir) continue; var_dump($list);
}
//exit(0);
/*
array(4) {
[2]=>
string(10) "file_0.img"
[3]=>
string(10) "file_1.img"
[4]=>
string(10) "file_2.img"
[5]=>
string(7) "results"
}
*/
// should not mix files and directories - should be same level
foreach ($list as $index=>$item) {
if (is_dir("$path/$item")) {
if ($item == $processed_subdir)
continue;
$sublist = preg_grep('/^([^.])/', scandir("$path/$item")); $sublist = preg_grep('/^([^.])/', scandir("$path/$item"));
foreach($sublist as $subitem){ sort($sublist);
if (split_file("$path/$item","$subitem","../$destination",$add_to_chn)==0){ foreach ($sublist as $file_index => $subitem) {
if ($move_processed){ $next_file = ($file_index < (count($sublist) -1)) ? $sublist[$file_index + 1] : "";
if (!is_dir("$path/$item/$processed_subdir")){ if (split_file("$path/$item", "$subitem", "../$destination", $add_to_chn, $next_file) == 0) {
mkdir("$path/$item/$processed_subdir",0777); if ($move_processed) {
if (! is_dir("$path/$item/$processed_subdir")) {
mkdir("$path/$item/$processed_subdir", 0777);
} }
rename("$path/$item/$subitem","$path/$item/$processed_subdir/$subitem"); rename("$path/$item/$subitem", "$path/$item/$processed_subdir/$subitem");
} }
} }
} }
}else{ }else{
if (split_file("$path","$item","$destination",$add_to_chn)==0){ $next_file = ($index < (count($list)-1)) ? $list[$index + 1] : "";
if (split_file("$path","$item","$destination",$add_to_chn, $next_file)==0){
if ($move_processed){ if ($move_processed){
if (!is_dir("$path/$processed_subdir")){ if (!is_dir("$path/$processed_subdir")){
mkdir("$path/$processed_subdir",0777); mkdir("$path/$processed_subdir",0777);
...@@ -145,89 +178,173 @@ foreach ($list as $item) { ...@@ -145,89 +178,173 @@ foreach ($list as $item) {
} }
//./extract_images_tiff.php path=/home/eyesis/captures/tests/jp4/ dest_path=results chn_offs=16 //./extract_images_tiff.php path=/home/eyesis/captures/tests/jp4/ dest_path=results chn_offs=16
function split_file($path,$file,$destination,$add_to_chn=-1){ function split_file($path, $file, $destination, $add_to_chn = - 1, $next_file = "")
{
if ($next_file && !is_file("$path/$next_file")){
if ((DEBUG_LEVEL >= 1)) {
printf("Ignoring $next_file - it is a DIRECTORY\n");
}
$next_file="";
}
// global $startMarkerWithExif = START_JP; // START_TIFF // global $startMarkerWithExif = START_JP; // START_TIFF
global $chunksize; global $chunksize;
global $input_exts; global $input_exts;
global $forced_ext; global $forced_ext;
if (in_array(get_ext("$path/$file"),$input_exts)) { if (in_array(get_ext("$path/$file"), $input_exts)) {
echo date(DATE_RFC2822).": Splitting $path/$file, results dir: $path/$destination\n"; $next_path = $next_file ? "$path/$next_file" : "";
//split_mov("$path",$file,$destination,$extension,$startMarkerWithExif,$chunksize); echo date(DATE_RFC2822) . ": Splitting $path/$file (next is $next_path):, results dir: $path/$destination\n";
// split_mov("$path",$file,$destination,$extension,$startMarkerWithExif,$chunksize);
$file_type = 0; // JP4/JPEG $file_type = 0; // JP4/JPEG
$markers=array(); $markers = array();
$offset =0; $offset = 0;
$f=fopen("$path/$file","r"); $f = fopen("$path/$file", "r");
$s = fread($f,$chunksize); $s = fread($f, $chunksize);
$pos_jp = strpos($s,START_JP); $pos_jp = strpos($s, START_JP);
$pos_tiff = strpos($s,START_TIFF); $pos_tiff = strpos($s, START_TIFF);
if (($pos_jp === false) && ($pos_tiff === false)) { if (($pos_jp === false) && ($pos_tiff === false)) {
print ("None of TIFF (".bin2hex(START_TIFF).") or JP4/JPEG (".bin2hex(START_JP).") markers found in the first ".$chunksize." bytes of the file $path/$file\n"); if (strlen($s) == $chunksize) {
return -1; print("None of TIFF (" . bin2hex(START_TIFF) . ") or JP4/JPEG (" . bin2hex(START_JP) . ") markers found in the first " . $chunksize . " bytes of the file $path/$file\n");
} else {
print("None of TIFF (" . bin2hex(START_TIFF) . ") or JP4/JPEG (" . bin2hex(START_JP) . ") markers found in the remaining " . strlen($s) . " bytes of the file $path/$file\n");
}
return - 1;
} }
if ($pos_jp === false) { if ($pos_jp === false) {
$file_type = 1; // tiff $file_type = 1; // tiff
} else if (($pos_tiff !== false) && ($pos_tiff < $pos_jp)){ // reducing probability of stray START_JP } else if (($pos_tiff !== false) && ($pos_tiff < $pos_jp)) { // reducing probability of stray START_JP
$file_type = 1; // tiff $file_type = 1; // tiff
} else { } else {
$file_type = 0; // jpeg/jp4 $file_type = 0; // jpeg/jp4
} }
$startMarkerWithExif = $file_type? START_TIFF: START_JP; // START_TIFF $startMarkerWithExif = $file_type ? START_TIFF : START_JP; // START_TIFF
echo "Detected image type ".($file_type?'TIFF':'JPEG/JP4').".\n"; echo "Detected image type " . ($file_type ? 'TIFF' : 'JPEG/JP4') . ".\n";
$forced_ext = $file_type?'tiff':''; // was ".tiff":"" $forced_ext = $file_type ? 'tiff' : ''; // was ".tiff":""
fclose($f); // not sure if feof is cleraed by fseek, closing/ reopening fclose($f); // not sure if feof is cleraed by fseek, closing/ reopening
$f=fopen("$path/$file","r"); $f = fopen("$path/$file", "r");
//first scan // first scan
while (!feof($f)) { while (! feof($f)) {
$pos=0; $pos = 0;
$index=0; $index = 0;
fseek($f,$offset); fseek($f, $offset); // will read from the last
$s = fread($f,$chunksize); $s = fread($f, $chunksize);
while(true){ if ($next_file && (strlen($s) < $chunksize)) { // special case, try to add beginning of the next file
$pos=strpos($s,$startMarkerWithExif,$pos); printf("Looking for the marker in the file split, using %s", $next_file);
if ($pos === false) break; $f1 = fopen("$path/$next_file", "r");
// echo count($markers).': '.strval(($offset+$pos)% 4096) .' ('.strval($offset+$pos).')'; // debugging $s1 = fread($f1, strlen($startMarkerWithExif) - 1);
$markers[count($markers)]=$offset+$pos; fclose($f1);
$s .= $s1;
$pos++; }
while (true) {
$pos = strpos($s, $startMarkerWithExif, $pos);
if ($pos === false) {
break;
}
if (REQUIRE_EXIF && ($file_type == 0) && (strpos($s, START_EXIF, $pos) != ($pos + EXIF_OFFSET))) { // JPEG/JP4
printf("Could not find required by REQUIRE_EXIF==%d Exif signature at offset %d. Skipping this marker (false positive?)\n", REQUIRE_EXIF, START_EXIF);
} else {
$markers[count($markers)] = $offset + $pos;
}
$pos ++;
} }
$offset+=(strlen($s)-strlen($startMarkerWithExif)+1); // so each marker will appear once // steps back by needle length-1, so partial markers will not be lost
$offset += (strlen($s) - strlen($startMarkerWithExif) + 1); // so each marker will appear once
} }
$markers[count($markers)]=$offset+strlen($s); // full length of the file $markers[count($markers)] = $offset + strlen($s); // full length of the file
$tmp_name = $file_type? "$path/$destination/header.tmp": "$path/$destination/image.tmp";
echo " images found: ".(count($markers)-1)."\n"; echo " images found: " . (count($markers) - 1) . "\n";
//second scan // second scan
for ($i=0;$i<(count($markers)-1);$i++) { $dbg_last = 0;
for ($i = 0; $i < (count($markers) - 1); $i ++) {
if ((DEBUG_LEVEL >= 2)) {
if ($i == 0) {
printf("1.First image[%d], markers[%d]=0x%08x, markers[%d]=0x%08x, diff=0x%08x\n", $i, $i, $markers[$i], $i + 1, $markers[$i + 1], $markers[$i + 1] - $markers[$i]);
} else if ($i == (count($markers) - 2)) {
printf("1.Last image[%d], markers[%d]=0x%08x, markers[%d]=0x%08x, diff=0x%08x\n", $i, $i, $markers[$i], $i + 1, $markers[$i + 1], $markers[$i + 1] - $markers[$i]);
}
}
fseek($f, $markers[$i]);
$s = fread($f, $markers[$i + 1] - $markers[$i]);
if ((DEBUG_LEVEL >= 2)) {
if (($i == (count($markers) - 2)) || ($i == 0)) {
printf("Read 0x%08x (wanted 0x%08x) bytes\n", strlen($s), $markers[$i + 1] - $markers[$i]);
}
}
$image_length = - 1;
fseek($f,$markers[$i]); if ($file_type == 0) { // JPEG/JP4
$s = fread($f,$markers[$i+1]-$markers[$i]); // find end marker
$end_pos = false;
if (strlen($s) >= MIN_JP_LEN) {
$end_pos = strpos($s, END_JP, MIN_JP_LEN); // PHP Warning: strpos(): Offset not contained in string in /home/elphel/git/elphel-tools-x393/extract_images_tiff.php on line 254
}
if (!$end_pos && $next_path) {// try reading from the next file define('MAX_IMG_LEN', 0x1000000); // maximal image size (16M) to look for the end marker
$f1 = fopen($next_path, "r");
$s .= fread($f1, MAX_IMG_LEN); // beginning of the next file
fclose($f1);
$end_pos = strpos($s, END_JP, MIN_JP_LEN); // PHP Warning: strpos(): Offset not contained in string in /home/elphel/git/elphel-tools-x393/extract_images_tiff.php on line 254
if ($end_pos && (DEBUG_LEVEL >= 1)) {
printf ("Found image continued in the next file %s\n", $next_path);
}
}
if ($end_pos) {
$end_pos+=strlen(END_JP);
file_put_contents($tmp_name, substr($s, 0, $end_pos));
$image_length = 0; // will not try to get it
$result_name = elphel_specific_result_name($tmp_name, $image_length, $add_to_chn);
if (DEBUG_OUT_FILES) {
$result_name = substr($result_name,0,strlen($result_name)-4).'-'.strval($i).'.jp4';
}
if ((DEBUG_LEVEL >= 3)) {
printf("%05d: 0x%08x l=0x%06x (0x%06x) %s\n",$i, $markers[$i], $end_pos, $markers[$i + 1] - $markers[$i], "$path/$destination/$result_name");
}
$dest_image = "$path/$destination/$result_name"; // $result_name may now include "/"
$dest_set_dir = dirname($dest_image);
if (! is_dir($dest_set_dir)) {
mkdir($dest_set_dir, 0777);
}
rename($tmp_name, $dest_image); // "$path/$destination/$result_name");
}
$tmp_name = "$path/$destination/header.tmp"; // image.tmp"; } else if ($file_type == 1) { // Tiff
// file_put_contents($tmp_name,$s); // file_put_contents($tmp_name,$s);
file_put_contents($tmp_name,substr($s, 0, MAX_HEAD_LEN)); // only save beginning of the file file_put_contents($tmp_name, substr($s, 0, MAX_HEAD_LEN)); // only save beginning of the file
// printf ("header length = %d\n", strlen(substr($s, 0, MAX_HEAD_LEN))); // printf ("header length = %d\n", strlen(substr($s, 0, MAX_HEAD_LEN)));
$image_length=-1;
$result_name = elphel_specific_result_name($tmp_name, $image_length, $add_to_chn); $result_name = elphel_specific_result_name($tmp_name, $image_length, $add_to_chn);
if ($image_length < 0){ if ($image_length < 0) {
printf ("Wrong image length from Exif header: %d", $image_length); printf("Wrong image length from Exif header: %d", $image_length);
exit(1); exit(1);
} }
$dest_image = "$path/$destination/$result_name"; // $result_name may now include "/" $dest_image = "$path/$destination/$result_name"; // $result_name may now include "/"
$dest_set_dir = dirname($dest_image); $dest_set_dir = dirname($dest_image);
if (!is_dir($dest_set_dir)){ if (! is_dir($dest_set_dir)) {
mkdir($dest_set_dir,0777); mkdir($dest_set_dir, 0777);
} }
// rename($tmp_name, $dest_image); // "$path/$destination/$result_name"); // rename($tmp_name, $dest_image); // "$path/$destination/$result_name");
file_put_contents($dest_image,substr($s, 0, $image_length)); file_put_contents($dest_image, substr($s, 0, $image_length));
if ((DEBUG_LEVEL >= 2)) {
if ($i == 0) {
file_put_contents($dest_image . ".test", substr($s, 0, $image_length));
printf("2.First image[%d] from img file - %s, length=0x%08x(%d), strlen(s)=%d (%d)\n", $i, $dest_image, $image_length, $image_length, strlen($s), strlen(substr($s, 0, $image_length)));
} else if ($i == (count($markers) - 2)) {
file_put_contents($dest_image . ".test", substr($s, 0, $image_length));
printf("2.Last image[%d] from img file - %s, length=0x%08x(%d), strlen(s)=%d (%d)\n", $i, $dest_image, $image_length, $image_length, strlen($s), strlen(substr($s, 0, $image_length)));
}
}
}
}
if (file_exists($tmp_name)){
unlink ($tmp_name);
} }
return 0; return 0;
}else{ } else {
return -1; return - 1;
} }
} }
...@@ -242,7 +359,9 @@ function elphel_specific_result_name($file, &$image_length, $add_to_chn=-10){ ...@@ -242,7 +359,9 @@ function elphel_specific_result_name($file, &$image_length, $add_to_chn=-10){
$exif = exif_read_data($file); // gets false find, what is wrong $exif = exif_read_data($file); // gets false find, what is wrong
// image size = ([StripOffsets] => 557)+ ([StripByteCounts] => 656640) // image size = ([StripOffsets] => 557)+ ([StripByteCounts] => 656640)
if ($image_length != 0) {
$image_length = intval($exif['StripOffsets']) + intval($exif['StripByteCounts']); $image_length = intval($exif['StripOffsets']) + intval($exif['StripByteCounts']);
}
$ext = elphel_specific_result_ext($exif,$forced_ext); $ext = elphel_specific_result_ext($exif,$forced_ext);
//converting GMT a local time GMT+7 //converting GMT a local time GMT+7
......
int_ssd_download.py
View file @ d5901897
...@@ -147,7 +147,7 @@ for i in range(len(cams)): ...@@ -147,7 +147,7 @@ for i in range(len(cams)):
if args.n==0: if args.n==0:
data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end) data_size_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_end)
data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start) data_skip_blocks = pc.read_camogm_disk_file_blocks("/dev/"+p[1][0:-1]+"1", args.file_start)
data_size_blocks -= data_skip_blocks data_size_blocks -= data_skip_blocks # before it included skipped !
else: else:
data_size_blocks = args.n data_size_blocks = args.n
data_skip_blocks = args.skip data_skip_blocks = args.skip
......
x393.py
View file @ d5901897
...@@ -260,7 +260,8 @@ class PC(): ...@@ -260,7 +260,8 @@ class PC():
if not os.path.isdir(dirname): if not os.path.isdir(dirname):
os.mkdir(dirname) os.mkdir(dirname)
for i in range(0,dl_n+dl_skip): for i in range(0,dl_n+dl_skip):
fname = dirname+"/"+"file_"+str(i)+".img" # fname = dirname+"/"+"file_"+str(i)+".img"
fname = "%s/file_%03d.img" %(dirname, i) #dirname+"/"+"file_"+str(i)+".img"
skip = i*(dl_bc-1) skip = i*(dl_bc-1)
if i>=dl_skip: if i>=dl_skip:
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip)) shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(dl_bs)+"M count="+str(dl_bc)+" skip="+str(skip))
...@@ -280,12 +281,12 @@ class PC(): ...@@ -280,12 +281,12 @@ class PC():
os.mkdir(dirname) os.mkdir(dirname)
num_file = 0 num_file = 0
# optional first file to align skip to chunk_blocks, 1 block at a time # optional first file to align skip to chunk_blocks, 1 block at a time
if (blocks_skip > 0) and ((blocks_skip // chunk_blocks) > 0): if (blocks_skip > 0) and ((blocks_skip % chunk_blocks) > 0):
bwrite = chunk_blocks - (blocks_skip // chunk_blocks) bwrite = chunk_blocks - (blocks_skip % chunk_blocks)
if (bwrite > blocks_load): if (bwrite > blocks_load):
bwrite = blocks_load bwrite = blocks_load
fname = dirname+"/"+"file_"+str(num_file)+".img" fname = dirname+"/"+"file_"+str(num_file)+".img"
print("Aligning skip to chunks, writing %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname)) print("Aligning skip to chunks, downloading %d %d-byte blocks (skipping %d blocks) to %s"%(bwrite, block_size, blocks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip)) shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(bwrite)+" skip="+str(blocks_skip))
blocks_skip += bwrite blocks_skip += bwrite
blocks_load -= bwrite blocks_load -= bwrite
...@@ -297,7 +298,7 @@ class PC(): ...@@ -297,7 +298,7 @@ class PC():
if (chunks_write > file_chunks): if (chunks_write > file_chunks):
chunks_write = file_chunks chunks_write = file_chunks
fname = dirname+"/"+"file_"+str(num_file)+".img" fname = dirname+"/"+"file_"+str(num_file)+".img"
print("Writing %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname)) print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)) shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
bwrite = chunks_write * chunk_blocks bwrite = chunks_write * chunk_blocks
blocks_skip += bwrite blocks_skip += bwrite
...@@ -306,7 +307,7 @@ class PC(): ...@@ -306,7 +307,7 @@ class PC():
# optionally write the remainder (< chunk), 1 block at a time # optionally write the remainder (< chunk), 1 block at a time
if (blocks_load > 0): if (blocks_load > 0):
fname = dirname+"/"+"file_"+str(num_file)+".img" fname = dirname+"/"+"file_"+str(num_file)+".img"
print("Writing last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname)) print("Downloading last %d %d-byte blocks, skipping %d blocks to %s"%(blocks_load, block_size, blocks_skip, fname))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip)) shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(block_size)+" count="+str(blocks_load)+" skip="+str(blocks_skip))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment