added script to track used files for CDT filter

.gitignore

@@ -6,4 +6,4 @@ html/*
 *.directory
 attic/*
 php
-
+sysroots

used_files.py

+#!/usr/bin/env python
+# encoding: utf-8
+from __future__ import division
+from __future__ import print_function
+"""
+# @file used_files.py
+# @brief Extract file access data after build, modify CDT project configuration
+# (.cproject) accordingly
+# @copyright Copyright (C) 2016, Elphel.inc.
+# @param <b>License</b>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+@author:     Andrey Filippov
+@license:    GPLv3.0+
+@contact:    andrey@elphel.coml
+@deffield    updated: Updated
+"""
+__author__ = "Andrey Filippov"
+__copyright__ = "Copyright 2016, Elphel, Inc."
+__license__ = "GPL"
+__version__ = "3.0+"
+__maintainer__ = "Andrey Filippov"
+__email__ = "andrey@elphel.com"
+__status__ = "Development"
+
+import sys
+import os
+import time
+import xml.etree.ElementTree as ET
+'''
+TODO:Automate, find out why separate touch commands are needed
+Run this program twice:
+1-st run ./used_files.py linux -1
+and save shown timestamp
+Then run (some mystery here)
+touch src/drivers/ata/ahci_elphel.c
+touch src/drivers/elphel/sensor_common.c
+Wait 5 seconds and run (in a different console with appropriate sourcing)
+bitbake linux-xlnx -c compile -f 
+Then again
+./used_files.py linux <timestamp_from_the_first_run>
+If somethong went wrong you will need to resore .cproject from eclipse_project_setup directory
+
+For php:
+bitbake php -c cleansstate
+bitbake php -c unpack -f
+bitbake php -c configure -f
+./used_files.py php -1
+bitbake php -c compile -f
+./used_files.py php 1471044836.8
+
+Doing:
+bitbake php -c compile -f
+./used_files.py php -1
+bitbake php -c compile -f
+./used_files.py php 1471044836.8
+Corrupts make process
+
+'''
+def file_tree(flist): # Each file in list is a file, no directories
+    ftree={}
+    for p in flist:
+        node = ftree
+        seg_list=p.split(os.sep)
+        last_i=len(seg_list)-1
+        for i,segm in enumerate(seg_list):
+            if not segm in node:
+                if i == last_i:
+                    node[segm] = None
+                else:
+                    node[segm] = {}
+            node=node[segm]        
+                        
+    return ftree                   
+
+def exclude_list(ftree, flist):
+    mark = "*" # no file/dir name can be "*" 
+    def list_tree_recursive(root):
+        rslt = []
+        if not mark in root:
+            return [[""]] # convert to trailing "/" for directories
+        for n in root:
+            if not n == mark:
+                if root[n] is None:
+                    rslt.append([n])
+                else:
+                    
+                    for l in list_tree_recursive(root[n]):
+                        rslt.append([n]+l)
+        return rslt
+                        
+    ftree[mark]=None # mark top level dir                                         
+    for p in flist:
+        node = ftree
+        for segm in p.split(os.sep)[:-1]:
+            node=node[segm]
+            node[mark]=None # [mark] means used in flist
+        del node[p.split(os.sep)[-1]]
+    #print (ftree)
+#    for k in ftree:
+#       print(k) 
+    #Now prune unused directories
+    #prune_recursive(ftree) # (assuming root is used)
+    # now create list
+    files_list_list = list_tree_recursive(ftree)
+#    print (files_list_list)
+    #converrt to file paths
+    pl = []
+    for l in files_list_list:
+        pl.append(os.path.join(*(l[1:])))
+    pl = sorted (pl)
+    return pl     
+    
+            
+def proc_tree():
+    DEBUG = True
+    extensions =    [".h",".c",".cpp"]
+    exclude_start = ["linux"+os.sep+"scripts"+os.sep,"linux"+os.sep+"source"+os.sep+"scripts"+os.sep]
+    delta_t = 3 # seconds
+    try:
+        root_path = sys.argv[1]
+    except:
+        print ("Calling %s <root directory path> [timestamp]"%(os.path.basename(sys.argv[0])))
+    try:
+        start_time = float(sys.argv[2])
+    except:
+        start_time = 0.0
+
+    touch_files= start_time < 0.0   
+    print ("root_path = %s"%(root_path))
+#    root_path = "/home/eyesis/git/poky/linux-elphel/linux/"    
+    lstFiles = []
+    # Append  files to a list
+    for path, _, files in os.walk(root_path, followlinks = True):
+        for f in files:
+            for ext in extensions:
+                if f.endswith(ext):
+                    lstFiles.append(os.path.join(path, f))
+                    break
+     
+    all_tree= file_tree(sorted(lstFiles))
+    include_lst=[]        
+    lst_a = []
+    latest_at=0
+    for p in lstFiles:
+        if touch_files:
+            if  os.path.islink(p):
+                os.utime(os.path.realpath(p), None)
+            else:    
+                os.utime(p, None)
+        else:        
+#           at = time.ctime(os.stat(p).st_atime)
+            at = os.stat(p).st_atime
+            l = None
+            if  os.path.islink(p):
+                l = os.path.realpath(p)
+                at = os.stat(l).st_atime
+            latest_at = max((latest_at,at))    
+            if at > (start_time + delta_t):
+                #Scripts/lexers result in problems
+                exclude=False
+                for exStr in exclude_start:
+                    if p.startswith(exStr):
+                        exclude=True
+                        break
+                if exclude:
+                    break        
+                #exclude_start
+                lst_a.append([p,at,l])
+                include_lst.append(p)
+    if touch_files:
+        print (len(lstFiles), "last time = ", time.time())
+        return    
+            
+    excluding = exclude_list(all_tree, include_lst)        
+#    print (all_tree)
+#    print (sorted(include_lst))
+#    print ("|".join(excluding))
+    if DEBUG:        
+        with open("all_sources.lst","w" ) as f:
+            for p in sorted(lstFiles):
+                at = os.stat(p).st_atime
+                lnk=""
+                if  os.path.islink(p):
+                    at = os.stat(os.path.realpath(p)).st_atime
+                    lnk = os.path.realpath(p)
+                print (p,at,lnk, file=f)
+        with open("excluding.lst","w" ) as f:
+            for p in excluding:
+                print (p, file=f)
+#    include_tree= file_tree(sorted(include_lst))
+#    print(include_tree)
+    root_dir=include_lst[0].split(os.sep)[0]
+    print ("root_dir=",root_dir)
+    
+    xml= ET.parse(".cproject")
+    root=xml.getroot()
+#    for child in root:
+#        print(child.tag, child.attrib)
+
+    for child in root.iter('sourceEntries'):
+        for gchild in child:
+            print(gchild.tag)
+    
+    for child in root.iter('sourceEntries'):
+        for gchild in child:
+            if gchild.tag == 'entry':
+                attr = gchild.attrib
+                try:
+                    if (attr['kind'] ==  'sourcePath') and (attr['name'] ==  root_dir):
+                        child.remove (gchild)
+                        print ("Removed existing entry ",gchild.tag)
+                        break
+                except:
+                    print ("error matching attributes for ",gchild.tag)
+                    pass
+        break #after first   'sourceEntries' - should be just one?      
+    ET.SubElement(child, 'entry', {"flags":"VALUE_WORKSPACE_PATH", "kind":"sourcePath", "name":root_dir, "excluding":"|".join(excluding)})
+                             
+    for child in root.iter('sourceEntries'):
+        for gchild in child:
+            print(gchild.tag)
+            
+    oneliner= ET.tostring(root)
+    #overwrites original .cproject, may change to somethong different
+    with open(".cproject", "wr") as f:
+        f.write("""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?>""")
+        f.write(oneliner)
+            
+    print (len(lstFiles), len(lst_a), "last access time = ",latest_at)
+                
+if __name__ == '__main__':
+    proc_tree()
\ No newline at end of file