cli mode

src/read_imu_log.php

@@ -7,24 +7,91 @@
  * LICENSE: AGPL, see http://www.gnu.org/licenses/agpl.txt
  * Copyright (C) 2016 Elphel, Inc.
  */
+ 
+set_time_limit(0);
 
-$hardcodeddir = "logs";
+$CLI = (php_sapi_name()=="cli");
+
+if ($CLI){
+  $_SERVER['REMOTE_ADDR'] = "localhost";
+  $_SERVER['SERVER_ADDR'] = "localhost";
+  $_GET['format'] = "csv";
+  $_GET['nth'] = 1;
+  $_GET['limit'] = -1;
+  
+  $_GET['nogui'] = true;
+ 
+  if (isset($argv[1])) $_GET['file'] = $argv[1];
+  else{
+    die(<<<TEXT
+    
+\033[91mERROR: Filename is not set.\033[0m 
+
+Command line usage examples:
+* Minimal:
+    \033[1;37m~$ php thisscript.php logfile > logfile.csv\033[0m
+* With a filter, IMU records only:
+    \033[1;37m~~$ php thisscript.php logfile 0x010 > logfile.csv\033[0m
+* With a filter, GPS NMEA GPRMC records ony:
+    \033[1;37m~~$ php thisscript.php logfile 0x001 > logfile.csv\033[0m
+* Filter bits:
+    External trigger source:
+      0x200
+    Image records:
+      0x100 channel 3
+      0x080 channel 2
+      0x040 channel 1
+      0x020 channel 0
+    IMU:
+      0x010
+    GPS records:
+      0x008 - NMEA GPVTG
+      0x004 - NMEA GPGSA
+      0x002 - NMEA GPGGA (have coordinates)
+      0x001 - NMEA GPRMC (have coordinates) 
+
+    
+TEXT
+    );
+  }
  
+  $_GET['record'] = 0;
+}
+
+$thisname = basename($_SERVER['SCRIPT_NAME']);
+$remoteaccess = ($_SERVER['REMOTE_ADDR']!=$_SERVER['SERVER_ADDR']);
+$hardcodeddir = "logs";
+
 if (isset($_GET['format'])){
   $format = $_GET['format'];
 }else{
   $format = "html";
 }
 
+if (isset($_GET['nth'])){
+  $nth = $_GET['nth']+0;
+}else{
+  $nth = 1;
+}
+
 if (isset($_GET['limit'])){
   $limit = $_GET['limit']+0;
 }else{
-  $limit = 0;
+  $limit = -1;
 }
 
 if (isset($_GET['nogui'])){
   $nogui = true;
+  if ($format=="csv"){
+    header("Content-Type: application/octet-stream");
+    header('Content-Disposition: attachment; filename='.basename($_GET['file']).".".$format);
+  }else{
+    header("Content-Type: application/xml");
+    echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+    echo "<Document>\n";
+  }
 }else{
+  //header("Content-Type: text/xml");
   $nogui = false;
 }
 
@@ -39,7 +106,7 @@ if (isset($_GET['list'])){
 }
 
 if (isset($_GET['file'])){
-  if ($_SERVER['REMOTE_ADDR']!=$_SERVER['SERVER_ADDR']){
+  if ($remoteaccess){
     $file = $hardcodeddir."/".basename($_GET['file']);
   }else{
     $file = $_GET['file'];
@@ -48,7 +115,7 @@ if (isset($_GET['file'])){
   if (is_file($file)){
     $numRecordsInFile=filesize($file)/64;
   }else{
-    //this is for $_SERVER['REMOTE_ADDR']==$_SERVER['SERVER_ADDR'] 
+    //for local access also check $hardcodeddir
     if (is_file($hardcodeddir."/".$file)){
       $file = $hardcodeddir."/".$file;
       $numRecordsInFile=filesize($file)/64;
@@ -62,6 +129,11 @@ if (isset($_GET['file'])){
   $init = false;
 }
 
+if ($CLI){
+  $_GET['nrecords'] = $numRecordsInFile;
+  if (isset($argv[2])) $_GET['filter'] = $argv[2];
+}
+
 if (isset($_GET['record']))   $record = $_GET['record']+0;
 else                          $record=0;
 
@@ -79,6 +151,11 @@ if ($nRecords>($numRecordsInFile-$record)) $nRecords= $numRecordsInFile-$record;
 
 if ($limit>0) $nRecords= $numRecordsInFile;
 
+if (isset($_GET['showall'])){
+  $limit = $numRecordsInFile;
+  $nRecords= $numRecordsInFile;
+}
+
 if (isset($_GET['filter'])) $filter= intval($_GET['filter'],0);
 else                        $filter= 0x3ff;
 
@@ -92,6 +169,8 @@ else                        $filter= 0x3ff;
 64 - ODOMETER (type 3)
 */
 
+if ($limit<0) $limit = $nRecords;
+
 if(!$nogui){
   //list available files
   echo html();
@@ -109,9 +188,10 @@ $sindex = $record;
 $eindex = $record+$nRecords;
 
 if ($format=="csv"){
-  echo "<pre>Filename,$file\n";
+  echo "Filename,$file\n";
   echo "Found Records,$numRecordsInFile\n";
   echo "Record filter,0x$filterhex\n";
+  echo "Show limit, $limit\n";
   echo "Start index,$sindex\n";
   echo "End index,$eindex\n\n";
 }else{
@@ -120,6 +200,7 @@ if ($format=="csv"){
 <tr><td>Filename</td><td>$file</td></tr>
 <tr><td>Found records</td><td>$numRecordsInFile</td></tr>
 <tr><td>Filter</td><td>$filterhex</td></tr>
+<tr><td>Show limit</td><td>$limit</td></tr>
 <tr><td>Start index</td><td>$sindex</td></tr>
 <tr><td>End index</td><td>$eindex</td></tr>
 </table>
@@ -164,23 +245,47 @@ imuLogParse($log_file,$record,$nRecords,$filter);
 
 die(0);
 
+$gc = array(
+  array(0,0,0,0),
+  array(0,0,0,0),
+  array(0,0,0,0),
+  array(0,0,0,0)
+);
+
+function global_counters($i,$j){
+  global $gc, $nth;
+  
+  if (($gc[$i][$j]%$nth)==0) {
+    $gc[$i][$j] = 1;
+    return true;
+  }else{
+    $gc[$i][$j]++;
+    return false;
+  }
+}
+
 function imuLogParse($handle,$record,$nSamples,$filter,$tryNumber=10000){
 
   global $timeShift;
   global $averageIMU;
   global $imuFieldOrder;
   global $format,$limit;
+  
+  global $gc;
     
-  $gpsFilter= $filter&0xf;
+  $gpsFilter = ($filter)&0xf;    //type=1
+  $imuFilter = ($filter>>4)&0x1; //type=0
+  $imgFilter = ($filter>>5)&0xf; //type=2
+  $extFilter = ($filter>>9)&0x1; //type=3
   
-  $typeFilter=(($gpsFilter!=0)?2:0) | ((($filter&0x10)!=0)?1:0) | ((($filter&0x20)!=0)?4:0) | ((($filter&0x20)!=0)?8:0);
+  $typeFilter = (($gpsFilter!=0)?2:0) | (($imuFilter!=0)?1:0) | ((($imgFilter)!=0)?4:0) | ((($extFilter)!=0)?8:0);
   
   $skip_imu_cols = true;
   $imu_cols_csv = "";
   $imu_cols_html_header = "";
   $imu_cols_html = "";
   
-  if ((($filter>>4)&0x1)==0x1) {
+  if ($imuFilter==0x1) {
     $skip_imu_cols = false;
     $imu_cols_csv_header = implode(",",$imuFieldOrder).",";
     for($i=0;$i<count($imuFieldOrder);$i++){
@@ -212,22 +317,43 @@ TEXT;
     echo "</tr>\n";
   }
   
+  $gc = array(
+    array(0,0,0,0),
+    array(0,0,0,0),
+    array(0,0,0,0),
+    array(0,0,0,0)
+  );
+  
   for ($nSample=0;$nSample<$nSamples;$nSample++) {
   
     $sample=fread($handle,64);
     $arr32=unpack('L*',$sample);
-    
     $time=(($arr32[1]&0xfffff)/1000000)+$arr32[2];
+    
     $type=$arr32[1]>>24;
-    $gps=($type==1)?($arr32[3]&0x3):0xf; // any if it is not GPS sample
     
-    if ((((1<<$type)&$typeFilter)!=0) && (($type!=1) || (((1<<$gps)&$gpsFilter)!=0))) {
-      
-      $limit--;
-      if ($limit==-1){
+    $show_record = false;
+    switch($type){
+      case 0:
+        if ($imuFilter!=0) $show_record = global_counters($type,0);
         break;
-      }
+      case 1:
+        if ((($gpsFilter>>($arr32[3]&0x3))&1)==1) $show_record = global_counters($type,$arr32[3]&0x3);
+        break;
+      case 2:
+        $subchannel = ($arr32[3] >> 24);
+        if ((($imgFilter>>$subchannel)&1)==1) $show_record = global_counters($type,$subchannel);
+        break;
+      case 3:
+        if (($extFilter&1)==1) $show_record = global_counters($type,0);
+        break;
+    }
+    
+    if ($show_record) {
       
+      $limit--;
+      if ($limit==-1) break;
+      //beginning
       if ($format=="csv"){
         printf("%d,%f,$type,",($record+$nSample),$time);
       }else{
@@ -284,8 +410,8 @@ TEXT;
           
         // Master (Sync) record
         case 2:
-          $masterTime=(($arr32[3]&0xfffff)/1000000)+$arr32[4];
           $subchannel = ($arr32[3] >> 24);
+          $masterTime=(($arr32[3]&0xfffff)/1000000)+$arr32[4];
           
           if ($format=="csv"){
             printf("$imu_cols_csv$subchannel,%f,".(gmdate(DATE_RFC850,$masterTime))."\n",$masterTime);
@@ -297,8 +423,6 @@ TEXT;
             printf("<td>%f</td>\n",$masterTime);
             echo "<td>".gmdate(DATE_RFC850,$masterTime)."</td>\n";
           }
-          
-          
           break;
           
         // Show hex data
@@ -323,6 +447,7 @@ TEXT;
   }
   if ($format!="csv"){
     echo "</table>\n";
+    echo "</Document>\n";
   }
 }
 
@@ -513,14 +638,14 @@ eg3. $GPVTG,t,T,,,s.ss,N,s.ss,K*hh
 function showlist(){
 
   global $hardcodeddir;
+  global $remoteaccess;
  
   $local = true;
  
   if (isset($_GET['file'])) $file = $_GET['file'];
    
-  if ($_SERVER['REMOTE_ADDR']!=$_SERVER['SERVER_ADDR']){
+  if ($remoteaccess){
     $dir = $hardcodeddir;
-    $local = false;
   }else{
     if (!isset($file)||($file=="")){
       $dir = ".";
@@ -540,8 +665,8 @@ function showlist(){
   
   $files = scandir($dir);
   
-  if ($dir=="."||(!$local)) $dirlink="";
-  else                      $dirlink="$dir/";
+  if ($dir=="."||($remoteaccess)) $dirlink="";
+  else                            $dirlink="$dir/";
   
   $res = "";
   
@@ -562,19 +687,29 @@ function showlist(){
 
 function html(){
 
-  global $file,$limit,$record,$nRecords;
+  global $file,$limit,$record,$nRecords,$filter,$nth;
   global $init;
 
+  $ins_filter = array();
+  
   if ($init) {
     $ins_file = $file;
     $ins_limit = $limit;
     $ins_rec = $record;
     $ins_nrec = $_GET['nrecords']+$record;
+    for($i=0;$i<10;$i++){
+      if (($filter>>$i)&1==1) $ins_filter[$i] = "checked";
+    }
+    $ins_nth = $nth;
   }else{
     $ins_file = "imu.log";
-    $ins_limit = 5000;
+    $ins_limit = 500;
     $ins_rec = 0;
     $ins_nrec = 5000;
+    for($i=0;$i<10;$i++){
+      $ins_filter[$i] = "checked";
+    }
+    $ins_nth = 1;
   }
   
   $js = js();
@@ -610,22 +745,22 @@ function html(){
         <tr>
           <td colspan=''>Record filter:&nbsp;
             <span title='External device'>EXT
-              <input id='filter_9' type='checkbox' checked onchange='getRqStr()'>
+              <input id='filter_9' type='checkbox' {$ins_filter[9]} onchange='getRqStr()' title='External source'>
             </span>&nbsp;&nbsp;
             <span title='Image trigger signal'>IMG
-              <input id='filter_8' type='checkbox' checked onchange='getRqStr()' title='Sensor port 3'>
-              <input id='filter_7' type='checkbox' checked onchange='getRqStr()' title='Sensor port 2'>
-              <input id='filter_6' type='checkbox' checked onchange='getRqStr()' title='Sensor port 1'>
-              <input id='filter_5' type='checkbox' checked onchange='getRqStr()' title='Sensor port 0'>
+              <input id='filter_8' type='checkbox' {$ins_filter[8]} onchange='getRqStr()' title='Sensor port 3'>
+              <input id='filter_7' type='checkbox' {$ins_filter[7]} onchange='getRqStr()' title='Sensor port 2'>
+              <input id='filter_6' type='checkbox' {$ins_filter[6]} onchange='getRqStr()' title='Sensor port 1'>
+              <input id='filter_5' type='checkbox' {$ins_filter[5]} onchange='getRqStr()' title='Sensor port 0'>
             </span>&nbsp;&nbsp;
             <span title='IMU'>IMU
-              <input id='filter_4' type='checkbox' checked onchange='getRqStr()'>
+              <input id='filter_4' type='checkbox' {$ins_filter[4]} onchange='getRqStr()'>
             </span>&nbsp;&nbsp;
             <span title='GPS'>GPS
-              <input id='filter_3' type='checkbox' checked onchange='getRqStr()' title='NMEA GPVTG'>
-              <input id='filter_2' type='checkbox' checked onchange='getRqStr()' title='NMEA GPGSA'>
-              <input id='filter_1' type='checkbox' checked onchange='getRqStr()' title='NMEA GPGGA'>
-              <input id='filter_0' type='checkbox' checked onchange='getRqStr()' title='NMEA GPRMC'>
+              <input id='filter_3' type='checkbox' {$ins_filter[3]} onchange='getRqStr()' title='NMEA GPVTG'>
+              <input id='filter_2' type='checkbox' {$ins_filter[2]} onchange='getRqStr()' title='NMEA GPGSA'>
+              <input id='filter_1' type='checkbox' {$ins_filter[1]} onchange='getRqStr()' title='NMEA GPGGA'>
+              <input id='filter_0' type='checkbox' {$ins_filter[0]} onchange='getRqStr()' title='NMEA GPRMC'>
             </span>
           </td>
         </tr>
@@ -640,14 +775,19 @@ function html(){
               <td></td>
               <td>End</td><td><input id='end' type='text' value='$ins_nrec' style='width:100px;text-align:right;' onchange='getRqStr()'></td>
             </tr>
+            <tr>
               <td><input type='checkbox' id='show_limit_toggle' checked onchange='getRqStr()'></td>
               <td>Show limit</td><td><input id='limit' type='text' value='$ins_limit' style='width:100px;text-align:right;' onchange='getRqStr()'></td>
             </tr>
+            <tr>
+              <td><input type='checkbox' id='show_nth_toggle' checked onchange='getRqStr()'></td>
+              <td>Show every</td><td><input id='nth' type='text' value='$ins_nth' style='width:100px;text-align:right;' onchange='getRqStr()'> <sup>th</sup> record</td>
+            </tr>
             </table>
           </td>
         </tr>
         <tr>
-          <td><button onclick='show()'>Show records</button>&nbsp;&nbsp;<span id='csvlink'></span></td>
+          <td><button onclick='show()'>Show records</button>&nbsp;&nbsp;<span id='csvlink'></span><span id='csvlink2'></span></td>
         </tr>
         </table>
       </td>
@@ -660,6 +800,7 @@ TEXT;
 }
 
 function js(){
+  global $thisname;
   global $init;
 
   if ($init) $insert = "show();";
@@ -677,8 +818,6 @@ function init(){
 
 function show(){
   console.log("show");
-    
-  report("status: waiting for response");
   
   var rqstr = getRqStr();
   
@@ -687,11 +826,9 @@ function show(){
 
   request.onreadystatechange = function() {
     if (this.readyState == 4 && this.status == 200) {      
-      var resp = this.responseText;
-      report("status: parsing response");
+      var resp = this.responseText;  
       clearInterval(loading_interval);
       document.getElementById("results").innerHTML = "<br/>"+resp;
-      report("<a href='"+rqstr+"'>current view</a>, <a href='"+rqstr+"&nogui"+"&format=csv"+"'>csv</a>, <a href='"+rqstr+"&nogui"+"&format=html"+"'>html</a>");
     }
   };
   
@@ -706,7 +843,7 @@ function show(){
 function getList(){
 
   var filename = document.getElementById("file").value;
-  var rqstr = "read_imu_log.php?list&nogui&file="+filename;
+  var rqstr = "$thisname?list&nogui&file="+filename;
   
   var request = new XMLHttpRequest();
   request.open('GET', rqstr, true);
@@ -731,7 +868,7 @@ function getRqStr(){
   var filename = document.getElementById("file").value;
   
   var filter = 0;
-  for(var i=0;i<7;i++){
+  for(var i=0;i<10;i++){
     bit = (document.getElementById("filter_"+i).checked)?1:0;
     filter += (bit<<i);
   }
@@ -745,6 +882,12 @@ function getRqStr(){
     limit = "&limit="+document.getElementById("limit").value;
   }
   
+  var nth = "";
+
+  if (document.getElementById("show_nth_toggle").checked){
+    nth = "&nth="+document.getElementById("nth").value;
+  }
+  
   var n = end - start;
   
   if (n<0) {
@@ -752,11 +895,11 @@ function getRqStr(){
     n = 1;
   }
   
-  var rqstr = "read_imu_log.php?file="+filename+"&record="+start+"&nrecords="+n+"&filter="+filter+limit;
+  var rqstr = "$thisname?file="+filename+"&record="+start+"&nrecords="+n+"&filter="+filter+limit+nth;
   
   report("");
   setTimeout(function(){
-    report("<a href='"+rqstr+"'>current view</a>, <a href='"+rqstr+"&nogui"+"&format=csv"+"'>csv</a>, <a href='"+rqstr+"&nogui"+"&format=html"+"'>html</a>");
+    report("<a href='"+rqstr+"'>share</a>, Download: <a href='"+rqstr+"&nogui"+"&format=csv'>csv</a>, <a href='"+rqstr+"&nogui"+"&format=csv&showall'>full csv</a>");
   },100);
   
   return rqstr;
@@ -798,22 +941,28 @@ var loading_interval;
 
 function loading(){
   console.log("loading");
-  var tmp = document.getElementById("csvlink").innerHTML;
+  var tmp = document.getElementById("csvlink2").innerHTML;
   if (tmp.length<2) tmp += ".";
   else              tmp = "";
-  document.getElementById("csvlink").innerHTML = tmp;
+  document.getElementById("csvlink2").innerHTML = tmp;
 }
 
 TEXT;
 }
 
 function help(){
+  global $thisname;
   global $nogui;
 
   $logslist = showlist();
   
   $help = <<<TEXT
   <span style='font-size:1.2em;'>$logslist</span>
+  <div style='font-size:1.2em;'>Source:
+    <ul>
+      <li><a href="?source" >This program source</a></li>
+    </ul>
+  </div>
   <div style='font-size:1.2em;'>Usage:
     <ul>
       <li>GUI:
@@ -822,39 +971,57 @@ function help(){
           &nbsp;&nbsp;&nbsp;&nbsp;<i>remote != server address</i> - http://thisscriptrootpath/logs/filename<br/>
           &nbsp;&nbsp;&nbsp;&nbsp;<i>remote == server address</i> - any relative/absolute path
           </li>
-          <li><b>Checkboxes</b> - checked = show</li>
+          <li><b>Record filter checkboxes</b> - checked = show</li>
           <li><b>Begin</b> - offset, record index in log</li>
           <li><b>End</b> - offset, record index in log</li>
           <li><b>Show limit</b> - Number of filtered records to show, if enabled overrides <b>End</b></li>
-          <li><b>current view, csv, html</b> - if any parameter changed a link to csv or html data is created/updated</li>
+          <li><button onclick='getList()'>List log files</button> :
+            <ul>
+              <li>remote access - list of files in <b>http://thisscriptrootpath/logs/</b></li>
+              <li>local access - list of files in <b>http://thisscriptrootpath/</b></li>
+            </ul>
+            <button onclick='show()'>Show records</button> - after a file is selected.<br/>
+            If a folder is selected - <button onclick='getList()'>List log files</button> again to scan inside.
+          </li>
+          <li><a href='#'>share</a> - open this page with the same parameters (auto <b>show records</b>)</li>
+          <li><a href='#'>csv</a> - download csv: filter + limit</li>
+          <li><a href='#'>csv full</a> - download csv: filter + no limit</li>
         </ul>
       </li>
       <br/>
-      <li>URL (see <b><i>current view, csv, html</i></b>):
+      <li>URL (see <b>share</b>,<b>csv</b>,<b>csv full</b>):
         <ul>
           <li>
-            <b>http://thisscriptrootpath/read_imu_log.php?file=..&record=..&nrecords=..&filter=..&limit=..&format=..</b><br/>
+            <b>http://thisscriptrootpath/$thisname?file=..&record=..&nrecords=..&filter=..&limit=..&format=..</b><br/>
             &nbsp;&nbsp;<b>file</b> - with path<br/>
             &nbsp;&nbsp;<b>format</b> - accepts 'csv' or 'html'<br/>
             &nbsp;&nbsp;<b>limit</b> - limit the displayed records<br/>
             &nbsp;&nbsp;<b>record</b> - starting record index, default = 0<br/>
             &nbsp;&nbsp;<b>nrecords</b> - number of records to parse, default = 5000<br/>
             &nbsp;&nbsp;<b>filter</b> - filter out types of displayed records:<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x40 - display external trigger records only<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x20 - display image records only<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x10 - display imu records only<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x200 - display external trigger records only<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x100 - display image records, channel 3 only<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x080 - display image records, channel 2 only<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x040 - display image records, channel 1 only<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x020 - display image records, channel 0 only<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x010 - display imu records only<br/>
             &nbsp;&nbsp;for gps records:<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x08 - display NMEA GPVTG records,<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x04 - display NMEA GPGSA records<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x02 - display NMEA GPGGA records (have coordinates)<br/>
-            &nbsp;&nbsp;&nbsp;&nbsp;0x01 - display NMEA GPRMC records (have coordinates)<br/>            
+            &nbsp;&nbsp;&nbsp;&nbsp;0x008 - display NMEA GPVTG records,<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x004 - display NMEA GPGSA records<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x002 - display NMEA GPGGA records (have coordinates)<br/>
+            &nbsp;&nbsp;&nbsp;&nbsp;0x001 - display NMEA GPRMC records (have coordinates)<br/>            
             &nbsp;&nbsp;default = 0x7f (display everything)<br/>
           </li>
         </ul>
       </li>
-      <li><a href="?source" >PHP source</a></li>
+      <li>Command line:
+        <ul>
+          <li><i>~$ php $thisname logfile filter [filter] > output.csv</i></li>
+        </ul>
+      </li>
     </ul>
     </div>
+    <br/><br/><br/>
 TEXT;
 
   if (!$nogui) $help = "`<br/>$help`";