Skip to content

E
elphel-tools-x393
Commit aeb17752 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

missing commits

parent dccc15bf
Hide whitespace changes
Inline Side-by-side
Showing with 464 additions and 1 deletion
extract_images_tiff_utc.php 0 → 100755
View file @ aeb17752
#!/usr/bin/env php
<?php
/**
* @file extract_images.php
* @brief split mov/bin/img's (by exif header) into single image files - looks one dir down from the specified path
* @copyright Copyright (C) 2017-2021 Elphel Inc.
* @author Elphel Inc. <support-list@support.elphel.com>
*
* @par <b>License</b>:
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
Found problem - does not process files that start in one file and end in another?
*/
define('START_TIFF', hex2bin('4d4d002a')); // start of a TIFF, JP/JP4 also have this TIFF marker
define('START_JP', hex2bin('ffd8ffe1')); // start of JPEG/JP4, TIFF usually does not have JPEG/JP4 markers
define('START_EXIF', hex2bin("457869660000")); // Exif\0\0
define('EXIF_OFFSET', 6); // Exif start offset from the beginning of the file (from start of START_JP)
define('END_JP', hex2bin('ffd9')); // end of JPEG/JP4 file
define('MAX_HEAD_LEN', 0x300); // maximal header length that includes complete header
define('MIN_JP_LEN', 0x500); // skip from jpeg looking for end marker
define('MAX_IMG_LEN', 0x1000000); // maximal image size (16M) to look for the end marker
define('REQUIRE_EXIF', 1); // JPEG/JP4 must have Exif (more filtering, less false positives)
define('DEBUG_LEVEL', 0); //3); // Generate debug output
//define('DEBUG_OUT_FILES',DEBUG_LEVEL? 1 : 0 ); // 1 to add sequence number to result image names (make all unique)
define('DEBUG_OUT_FILES',DEBUG_LEVEL? 0 : 0 ); // 1 to add sequence number to result image names (make all unique)
//disable the default time limit for php scripts.
set_time_limit(0);
$chunksize=10000000; //10MB
//$startMarkerWithExif=chr(hexdec("4d")).chr(hexdec("4d")).chr(hexdec("00")).chr(hexdec("2a"));
//$startMarkerWithExif=chr(hexdec("ff")).chr(hexdec("d8")).chr(hexdec("ff")).chr(hexdec("e1"));
$input_exts = array("img","bin","mov");
// use current dir
$path=".";
$destination = "0";
$move_processed = false;
$processed_subdir = "processed";
$forced_ext = "";
$add_to_chn = -1; // do not use and do not create scene directories
function print_help(){
global $argv;
echo <<<"TXT"
Help:
* Usage:
~$ {$argv[0]} path=[path-to-dir] dest_path=[dest-subdir] move_processed=[move-processed-files] ext=[forced-ext] chn_offs=[add-to-chn]
where:
* path-to-dir - string - scan this path + 1 dir down
* dest-subdir - string - save results to "path-to-dir/dest-subdir/"
* move-processed-files - 0(default) or 1 - if not 1 - will not move the processed files
* forced-ext - string - override extensions from exifs with this one
* add-to-chn - integer - add to decoded channel number
* Examples:
** Split all *.img, *.bin and *.mov files in the current dir and 1 dir down, puts results to '0/':
~$ {$argv[0]}
** Split in /data/test + 1 dir down, create and move processed files to /data/test/processed for files in path and /data/test/any-found-subdir/processed for any files found in /data/test/any-found-subdir
~$ {$argv[0]} path=/data/test move_processed=1
** Split all *.img, *.bin and *.mov files in the current dir and 1 dir down, puts results to 'results/', override extensions with 'jpg':
~$ {$argv[0]} dest_path=results ext=jpg
TXT;
}
if ($argv){
foreach($argv as $k=>$v){
if ($k==0) continue;
$args = explode("=",$v);
if (isset($args[1])) $_GET[$args[0]] = $args[1];
if ($v=="-h"||$v=="--help"||$v=="help") {
print_help();
die();
}
}
}
if (isset($_GET['path'])){
$path=$_GET['path'];
}
// put results to $path/$destination/
// no need for argv
if (isset($_GET['dest_path'])){
$destination = $_GET['dest_path'];
}
if (isset($_GET['move_processed'])){
$move_processed = $_GET['move_processed'];
}
if ($move_processed=="1"){
$move_processed = true;
}else{
$move_processed = false;
}
// enforce extension
if (isset($_GET['ext'])){
$forced_ext = $_GET['ext'];
}
if (isset($_GET['chn_offs'])){
$add_to_chn = (integer) $_GET['chn_offs'];
}
$list = preg_grep('/^([^.])/', scandir($path));
sort($list); // also re-indexes
//$list=array();
if (!is_dir("$path/$destination")) mkdir("$path/$destination",0777);
if ((DEBUG_LEVEL >= 2)) {
var_dump($list);
}
//exit(0);
/*
array(4) {
[2]=>
string(10) "file_0.img"
[3]=>
string(10) "file_1.img"
[4]=>
string(10) "file_2.img"
[5]=>
string(7) "results"
}
*/
// should not mix files and directories - should be same level
foreach ($list as $index=>$item) {
if (is_dir("$path/$item")) {
if ($item == $processed_subdir)
continue;
$sublist = preg_grep('/^([^.])/', scandir("$path/$item"));
sort($sublist);
foreach ($sublist as $file_index => $subitem) {
$next_file = ($file_index < (count($sublist) -1)) ? $sublist[$file_index + 1] : "";
if (split_file("$path/$item", "$subitem", "../$destination", $add_to_chn, $next_file) == 0) {
if ($move_processed) {
if (! is_dir("$path/$item/$processed_subdir")) {
mkdir("$path/$item/$processed_subdir", 0777);
}
rename("$path/$item/$subitem", "$path/$item/$processed_subdir/$subitem");
}
}
}
}else{
$next_file = ($index < (count($list)-1)) ? $list[$index + 1] : "";
if (split_file("$path","$item","$destination",$add_to_chn, $next_file)==0){
if ($move_processed){
if (!is_dir("$path/$processed_subdir")){
mkdir("$path/$processed_subdir",0777);
}
rename("$path/$item","$path/$processed_subdir/$item");
}
}
}
}
//./extract_images_tiff.php path=/home/eyesis/captures/tests/jp4/ dest_path=results chn_offs=16
function split_file($path, $file, $destination, $add_to_chn = - 1, $next_file = "")
{
if ($next_file && !is_file("$path/$next_file")){
if ((DEBUG_LEVEL >= 1)) {
printf("Ignoring $next_file - it is a DIRECTORY\n");
}
$next_file="";
}
// global $startMarkerWithExif = START_JP; // START_TIFF
global $chunksize;
global $input_exts;
global $forced_ext;
if (in_array(get_ext("$path/$file"), $input_exts)) {
$next_path = $next_file ? "$path/$next_file" : "";
echo date(DATE_RFC2822) . ": Splitting $path/$file (next is $next_path):, results dir: $path/$destination\n";
// split_mov("$path",$file,$destination,$extension,$startMarkerWithExif,$chunksize);
$file_type = 0; // JP4/JPEG
$markers = array();
$offset = 0;
$f = fopen("$path/$file", "r");
$s = fread($f, $chunksize);
$pos_jp = strpos($s, START_JP);
$pos_tiff = strpos($s, START_TIFF);
if (($pos_jp === false) && ($pos_tiff === false)) {
if (strlen($s) == $chunksize) {
print("None of TIFF (" . bin2hex(START_TIFF) . ") or JP4/JPEG (" . bin2hex(START_JP) . ") markers found in the first " . $chunksize . " bytes of the file $path/$file\n");
} else {
print("None of TIFF (" . bin2hex(START_TIFF) . ") or JP4/JPEG (" . bin2hex(START_JP) . ") markers found in the remaining " . strlen($s) . " bytes of the file $path/$file\n");
}
return - 1;
}
if ($pos_jp === false) {
$file_type = 1; // tiff
} else if (($pos_tiff !== false) && ($pos_tiff < $pos_jp)) { // reducing probability of stray START_JP
$file_type = 1; // tiff
} else {
$file_type = 0; // jpeg/jp4
}
$startMarkerWithExif = $file_type ? START_TIFF : START_JP; // START_TIFF
echo "Detected image type " . ($file_type ? 'TIFF' : 'JPEG/JP4') . ".\n";
$forced_ext = $file_type ? 'tiff' : ''; // was ".tiff":""
fclose($f); // not sure if feof is cleraed by fseek, closing/ reopening
$f = fopen("$path/$file", "r");
// first scan
while (! feof($f)) {
$pos = 0;
$index = 0;
fseek($f, $offset); // will read from the last
$s = fread($f, $chunksize);
if ($next_file && (strlen($s) < $chunksize)) { // special case, try to add beginning of the next file
printf("Looking for the marker in the file split, using %s", $next_file);
$f1 = fopen("$path/$next_file", "r");
$s1 = fread($f1, strlen($startMarkerWithExif) - 1);
fclose($f1);
$s .= $s1;
}
while (true) {
$pos = strpos($s, $startMarkerWithExif, $pos);
if ($pos === false) {
break;
}
if (REQUIRE_EXIF && ($file_type == 0) && (strpos($s, START_EXIF, $pos) != ($pos + EXIF_OFFSET))) { // JPEG/JP4
printf("Could not find required by REQUIRE_EXIF==%d Exif signature at offset %d. Skipping this marker (false positive?)\n", REQUIRE_EXIF, START_EXIF);
} else {
$markers[count($markers)] = $offset + $pos;
}
$pos ++;
}
// steps back by needle length-1, so partial markers will not be lost
$offset += (strlen($s) - strlen($startMarkerWithExif) + 1); // so each marker will appear once
}
$markers[count($markers)] = $offset + strlen($s); // full length of the file
$tmp_name = $file_type? "$path/$destination/header.tmp": "$path/$destination/image.tmp";
echo " images found: " . (count($markers) - 1) . "\n";
// second scan
$dbg_last = 0;
for ($i = 0; $i < (count($markers) - 1); $i ++) {
if ((DEBUG_LEVEL >= 2)) {
if ($i == 0) {
printf("1.First image[%d], markers[%d]=0x%08x, markers[%d]=0x%08x, diff=0x%08x\n", $i, $i, $markers[$i], $i + 1, $markers[$i + 1], $markers[$i + 1] - $markers[$i]);
} else if ($i == (count($markers) - 2)) {
printf("1.Last image[%d], markers[%d]=0x%08x, markers[%d]=0x%08x, diff=0x%08x\n", $i, $i, $markers[$i], $i + 1, $markers[$i + 1], $markers[$i + 1] - $markers[$i]);
}
}
fseek($f, $markers[$i]);
if ((DEBUG_LEVEL >= 1)) {
printf("marker: %6d, position 0x%x\n",$i,$markers[$i]);
}
$s = fread($f, $markers[$i + 1] - $markers[$i]);
if ((DEBUG_LEVEL >= 2)) {
if (($i == (count($markers) - 2)) || ($i == 0)) {
printf("Read 0x%08x (wanted 0x%08x) bytes\n", strlen($s), $markers[$i + 1] - $markers[$i]);
}
}
$image_length = - 1;
if ($file_type == 0) { // JPEG/JP4
// find end marker
$end_pos = false;
if (strlen($s) >= MIN_JP_LEN) {
$end_pos = strpos($s, END_JP, MIN_JP_LEN); // PHP Warning: strpos(): Offset not contained in string in /home/elphel/git/elphel-tools-x393/extract_images_tiff.php on line 254
}
if (!$end_pos && $next_path) {// try reading from the next file define('MAX_IMG_LEN', 0x1000000); // maximal image size (16M) to look for the end marker
$f1 = fopen($next_path, "r");
$s .= fread($f1, MAX_IMG_LEN); // beginning of the next file
fclose($f1);
$end_pos = strpos($s, END_JP, MIN_JP_LEN); // PHP Warning: strpos(): Offset not contained in string in /home/elphel/git/elphel-tools-x393/extract_images_tiff.php on line 254
if ($end_pos && (DEBUG_LEVEL >= 1)) {
printf ("Found image continued in the next file %s\n", $next_path);
}
}
if ($end_pos) {
$end_pos+=strlen(END_JP);
file_put_contents($tmp_name, substr($s, 0, $end_pos));
$image_length = 0; // will not try to get it
$result_name = elphel_specific_result_name($tmp_name, $image_length, $add_to_chn);
if (DEBUG_OUT_FILES) {
$result_name = substr($result_name,0,strlen($result_name)-4).'-'.strval($i).'.jp4';
}
if ((DEBUG_LEVEL >= 3)) {
printf("%05d: 0x%08x l=0x%06x (0x%06x) %s\n",$i, $markers[$i], $end_pos, $markers[$i + 1] - $markers[$i], "$path/$destination/$result_name");
}
$dest_image = "$path/$destination/$result_name"; // $result_name may now include "/"
$dest_set_dir = dirname($dest_image);
if (! is_dir($dest_set_dir)) {
mkdir($dest_set_dir, 0777);
}
rename($tmp_name, $dest_image); // "$path/$destination/$result_name");
}
} else if ($file_type == 1) { // Tiff
// read continuation if needed
//$s = fread($f, $markers[$i + 1] - $markers[$i]);
if ((strlen($s) < $markers[$i + 1] - $markers[$i]) && $next_path) {
$f1 = fopen($next_path, "r");
$s .= fread($f1, $markers[$i + 1] - $markers[$i] - strlen($s)); // beginning of the next file
fclose($f1);
if (DEBUG_LEVEL >= 0) {
printf("Got continuation of the image from the next file %s\n", $next_path);
}
}
// file_put_contents($tmp_name,$s);
file_put_contents($tmp_name, substr($s, 0, MAX_HEAD_LEN)); // only save beginning of the file
// printf ("header length = %d\n", strlen(substr($s, 0, MAX_HEAD_LEN)));
$result_name = elphel_specific_result_name($tmp_name, $image_length, $add_to_chn);
if ($image_length < 0) {
printf("Wrong image length from Exif header: %d", $image_length);
exit(1);
}
$dest_image = "$path/$destination/$result_name"; // $result_name may now include "/"
$dest_set_dir = dirname($dest_image);
if (! is_dir($dest_set_dir)) {
mkdir($dest_set_dir, 0777);
}
// rename($tmp_name, $dest_image); // "$path/$destination/$result_name");
file_put_contents($dest_image, substr($s, 0, $image_length));
if (DEBUG_LEVEL >= 2) {
if ($i == 0) {
file_put_contents($dest_image . ".test", substr($s, 0, $image_length));
printf("2.First image[%d] from img file - %s, length=0x%08x(%d), strlen(s)=%d (%d)\n", $i, $dest_image, $image_length, $image_length, strlen($s), strlen(substr($s, 0, $image_length)));
} else if ($i == (count($markers) - 2)) {
file_put_contents($dest_image . ".test", substr($s, 0, $image_length));
printf("2.Last image[%d] from img file - %s, length=0x%08x(%d), strlen(s)=%d (%d)\n", $i, $dest_image, $image_length, $image_length, strlen($s), strlen(substr($s, 0, $image_length)));
}
}
}
}
if (file_exists($tmp_name)){
unlink ($tmp_name);
}
return 0;
} else {
return - 1;
}
}
function elphel_specific_result_name($file, &$image_length, $add_to_chn=-10){
global $forced_ext;
// $add_to_chn<0 - do not use it and do not use scene directories
$use_scene_dirs = $add_to_chn >= 0;
if ($add_to_chn < 0){
$add_to_chn = 0;
}
$exif = exif_read_data($file); // gets false find, what is wrong
// image size = ([StripOffsets] => 557)+ ([StripByteCounts] => 656640)
if ($image_length != 0) {
$image_length = intval($exif['StripOffsets']) + intval($exif['StripByteCounts']);
}
$ext = elphel_specific_result_ext($exif,$forced_ext);
// date_default_timezone_set(string $timezoneId): bool // "UTC"
//converting GMT a local time GMT+7
date_default_timezone_set('UTC');
$timestamp_local=strtotime($exif['DateTimeOriginal']);/*-25200;*/ //
$subsecs = $exif['SubSecTimeOriginal']; //
$tmp = explode("_",$exif['Model']); //
if (count($tmp)==2){
if (trim($tmp[0])=="Eyesis4pi393"){
$model = intval(trim($tmp[1]));
$chn = intval($exif['PageNumber'])+1;
if ($model==1001) {
$k=$chn;
}else if ($model==1002) {
$k=$chn+4;
}else if ($model==1003) {
$k=$chn+6;
}
}else{
$ks = $exif['PageNumber'];
if (is_array($ks) && (count($ks) ==2)){
$k = $ks[0];
}else{
$k = intval($exif['PageNumber'])+1;
}
}
}else{
$k = intval($exif['PageNumber'])+1; //
}
if ($use_scene_dirs) {
$k += $add_to_chn;
return "{$timestamp_local}_{$subsecs}/{$timestamp_local}_{$subsecs}_$k.$ext";
} else {
return "{$timestamp_local}_{$subsecs}_$k.$ext";
}
}
function get_ext($filename) {
return pathinfo($filename, PATHINFO_EXTENSION);
}
/**
* read image format and return extension, elphel elphel_specific
* @param array $exif Array returned from the PHP's built-in exif_read_data function
* @param string $override_ext
* @return string extension - jpeg or jp4
*/
function elphel_specific_result_ext($exif,$override_ext=""){
//default value
$ext = "jpeg";
if ($override_ext==""){
if (isset($exif['MakerNote'][10])){
$record = ($exif['MakerNote'][10]>>4)&0xf;
if ($record==5) $ext = "jp4";
}
}else{
$ext = $override_ext;
}
return $ext;
}
?>
x393.py
View file @ aeb17752
...@@ -173,12 +173,14 @@ class Camera: ...@@ -173,12 +173,14 @@ class Camera:
shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'") shout(self.sshcmd+" 'echo 1 > /sys/devices/soc0/amba@0/80000000.elphel-ahci/load_module'")
def read_camogm_disk_file_blocks(self, part,fname="camogm.disk"): def read_camogm_disk_file_blocks(self, part,fname="camogm.disk"):
print("camera: read_camogm_disk_file_blocks() "+fname+ "self.scpcmd="+self.scpcmd)
result = 0 result = 0
tmp_mount_point = tempfile.mkdtemp() tmp_mount_point = tempfile.mkdtemp()
print(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point) print(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
shout(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point) shout(self.scpcmd+"/mnt/"+part+"/"+fname+" "+tmp_mount_point)
try: try:
with open (tmp_mount_point+"/"+fname, "r") as myfile: with open (tmp_mount_point+"/"+fname, "r") as myfile:
print ("Opened "+tmp_mount_point+"/"+fname)
data=myfile.readlines() data=myfile.readlines()
if len(data)==2: if len(data)==2:
l2 = data[1] l2 = data[1]
...@@ -210,11 +212,13 @@ class PC(): ...@@ -210,11 +212,13 @@ class PC():
# mounts partition (/dev/sd?1), reads camogm.disk file # mounts partition (/dev/sd?1), reads camogm.disk file
# returns the download size from raw partition ((/dev/sd?2)) # returns the download size from raw partition ((/dev/sd?2))
def read_camogm_disk_file(self,part,fname="camogm.disk"): def read_camogm_disk_file(self,part,fname="camogm.disk"):
print("read_camogm_disk_file() "+fname)
result = 0 result = 0
tmp_mount_point = tempfile.mkdtemp() tmp_mount_point = tempfile.mkdtemp()
print("mounting "+part+" to "+tmp_mount_point) print("mounting "+part+" to "+tmp_mount_point)
shout("sudo mount "+part+" "+tmp_mount_point) shout("sudo mount "+part+" "+tmp_mount_point)
print("mounted "+part+" to "+tmp_mount_point)
time.sleep(1)
try: try:
with open (tmp_mount_point+"/"+fname, "r") as myfile: with open (tmp_mount_point+"/"+fname, "r") as myfile:
data=myfile.readlines() data=myfile.readlines()
...@@ -256,13 +260,20 @@ class PC(): ...@@ -256,13 +260,20 @@ class PC():
Returns block offset from the start of partition to the current LBA pointer Returns block offset from the start of partition to the current LBA pointer
""" """
def read_camogm_disk_file_blocks(self,part,fname="camogm.disk"): def read_camogm_disk_file_blocks(self,part,fname="camogm.disk"):
# print("PC: read_camogm_disk_file_blocks() "+fname)
result = 0 result = 0
tmp_mount_point = tempfile.mkdtemp() tmp_mount_point = tempfile.mkdtemp()
print("mounting "+part+" to "+tmp_mount_point) print("mounting "+part+" to "+tmp_mount_point)
shout("sudo mount "+part+" "+tmp_mount_point) shout("sudo mount "+part+" "+tmp_mount_point)
print("mounted "+part+" to "+tmp_mount_point)
time.sleep(1)
# print ("Opening "+tmp_mount_point+"/"+fname)
# time.sleep(100);
try: try:
with open (tmp_mount_point+"/"+fname, "r") as myfile: with open (tmp_mount_point+"/"+fname, "r") as myfile:
# print("reading "+tmp_mount_point+"/"+fname)
data=myfile.readlines() data=myfile.readlines()
# print("len(data)= "+str(len(data)))
if len(data)==2: if len(data)==2:
l2 = data[1] l2 = data[1]
pointers = l2.split("\t") pointers = l2.split("\t")
...@@ -284,7 +295,9 @@ class PC(): ...@@ -284,7 +295,9 @@ class PC():
shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point) shout("scp root@"+ip+":/mnt/"+part+"/"+fname+" "+tmp_mount_point)
try: try:
with open (tmp_mount_point+"/"+fname, "r") as myfile: with open (tmp_mount_point+"/"+fname, "r") as myfile:
# print("reading "+tmp_mount_point+"/"+fname)
data=myfile.readlines() data=myfile.readlines()
# print("len(data)= "+str(len(data)))
if len(data)==2: if len(data)==2:
l2 = data[1] l2 = data[1]
pointers = l2.split("\t") pointers = l2.split("\t")
...@@ -371,6 +384,7 @@ class PC(): ...@@ -371,6 +384,7 @@ class PC():
# fname = dirname+"/"+"file_"+str(num_file)+".img" # fname = dirname+"/"+"file_"+str(num_file)+".img"
fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img" fname = "%s/file_%03d.img" %(dirname, num_file) #dirname+"/"+"file_"+str(num_file)+".img"
print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname)) print("Downloading %d %d-byte chunks, skipping %d chunks to %s"%(chunks_write, chunk_bytes, chunks_skip, fname))
print("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip)) shout("sudo dd if="+part+" "+" of="+fname+" bs="+str(chunk_bytes)+" count="+str(chunks_write)+" skip="+str(chunks_skip))
bwrite = chunks_write * chunk_blocks bwrite = chunks_write * chunk_blocks
blocks_skip += bwrite blocks_skip += bwrite
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment