used_files.py 9.33 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
#!/usr/bin/env python
# encoding: utf-8
from __future__ import division
from __future__ import print_function
"""
# @file used_files.py
# @brief Extract file access data after build, modify CDT project configuration
# (.cproject) accordingly
# @copyright Copyright (C) 2016, Elphel.inc.
# @param <b>License</b>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http:#www.gnu.org/licenses/>.

@author:     Andrey Filippov
@license:    GPLv3.0+
@contact:    andrey@elphel.coml
@deffield    updated: Updated
"""
__author__ = "Andrey Filippov"
__copyright__ = "Copyright 2016, Elphel, Inc."
__license__ = "GPL"
__version__ = "3.0+"
__maintainer__ = "Andrey Filippov"
__email__ = "andrey@elphel.com"
__status__ = "Development"

import sys
import os
import time
import xml.etree.ElementTree as ET
'''
TODO:Automate, find out why separate touch commands are needed
Run this program twice:
1-st run ./used_files.py linux -1
and save shown timestamp
Then run (some mystery here)
touch src/drivers/ata/ahci_elphel.c
touch src/drivers/elphel/sensor_common.c
Wait 5 seconds and run (in a different console with appropriate sourcing)
bitbake linux-xlnx -c compile -f 
Then again
./used_files.py linux <timestamp_from_the_first_run>
If somethong went wrong you will need to resore .cproject from eclipse_project_setup directory

For php:
bitbake php -c cleansstate
bitbake php -c unpack -f
bitbake php -c configure -f
./used_files.py php -1
bitbake php -c compile -f
./used_files.py php 1471044836.8

Doing:
bitbake php -c compile -f
./used_files.py php -1
bitbake php -c compile -f
./used_files.py php 1471044836.8
Corrupts make process

'''
def file_tree(flist): # Each file in list is a file, no directories
    ftree={}
    for p in flist:
        node = ftree
        seg_list=p.split(os.sep)
        last_i=len(seg_list)-1
        for i,segm in enumerate(seg_list):
            if not segm in node:
                if i == last_i:
                    node[segm] = None
                else:
                    node[segm] = {}
            node=node[segm]        
                        
    return ftree                   

def exclude_list(ftree, flist):
    mark = "*" # no file/dir name can be "*" 
    def list_tree_recursive(root):
        rslt = []
        if not mark in root:
            return [[""]] # convert to trailing "/" for directories
        for n in root:
            if not n == mark:
                if root[n] is None:
                    rslt.append([n])
                else:
                    
                    for l in list_tree_recursive(root[n]):
                        rslt.append([n]+l)
        return rslt
                        
    ftree[mark]=None # mark top level dir                                         
    for p in flist:
        node = ftree
        for segm in p.split(os.sep)[:-1]:
            node=node[segm]
            node[mark]=None # [mark] means used in flist
        del node[p.split(os.sep)[-1]]
    #print (ftree)
#    for k in ftree:
#       print(k) 
    #Now prune unused directories
    #prune_recursive(ftree) # (assuming root is used)
    # now create list
    files_list_list = list_tree_recursive(ftree)
#    print (files_list_list)
    #converrt to file paths
    pl = []
    for l in files_list_list:
        pl.append(os.path.join(*(l[1:])))
    pl = sorted (pl)
    return pl     
    
            
def proc_tree():
    DEBUG = True
    extensions =    [".h",".c",".cpp"]
    exclude_start = ["linux"+os.sep+"scripts"+os.sep,"linux"+os.sep+"source"+os.sep+"scripts"+os.sep]
    delta_t = 3 # seconds
    try:
        root_path = sys.argv[1]
    except:
        print ("Calling %s <root directory path> [timestamp]"%(os.path.basename(sys.argv[0])))
    try:
        start_time = float(sys.argv[2])
    except:
        start_time = 0.0

    touch_files= start_time < 0.0   
    print ("root_path = %s"%(root_path))
#    root_path = "/home/eyesis/git/poky/linux-elphel/linux/"    
    lstFiles = []
    # Append  files to a list
    for path, _, files in os.walk(root_path, followlinks = True):
        for f in files:
            for ext in extensions:
                if f.endswith(ext):
                    lstFiles.append(os.path.join(path, f))
                    break
     
    all_tree= file_tree(sorted(lstFiles))
    include_lst=[]        
    lst_a = []
    latest_at=0
    for p in lstFiles:
        if touch_files:
            if  os.path.islink(p):
                try:
                    os.utime(os.path.realpath(p), None)
                except:
                    print("missing linked file: %s"%(os.path.realpath(p)))
            else:    
                os.utime(p, None)
        else:        
#           at = time.ctime(os.stat(p).st_atime)
            try:
                at = os.stat(p).st_atime
                l = None
            except:
                at = 0    
            if  os.path.islink(p):
                try:
                    l = os.path.realpath(p)
                    at = os.stat(l).st_atime
                except:
                    at = 0 # missing file
            latest_at = max((latest_at,at))    
            if at > (start_time + delta_t):
                #Scripts/lexers result in problems
                exclude=False
                for exStr in exclude_start:
                    if p.startswith(exStr):
                        exclude=True
                        break
                if exclude:
                    break        
                #exclude_start
                lst_a.append([p,at,l])
                include_lst.append(p)
    if touch_files:
        print (len(lstFiles), "last time = ", time.time())
        return    
            
    excluding = exclude_list(all_tree, include_lst)        
#    print (all_tree)
#    print (sorted(include_lst))
#    print ("|".join(excluding))
    if DEBUG:        
        with open("all_sources.lst","w" ) as f:
            for p in sorted(lstFiles):
                try:
                    at = os.stat(p).st_atime
                except:
                    at = 0
                lnk=""
                if  os.path.islink(p):
                    try:
                        at = os.stat(os.path.realpath(p)).st_atime
                    except:
                        at = 0
                    lnk = os.path.realpath(p)
                print (p,at,lnk, file=f)
        with open("excluding.lst","w" ) as f:
            for p in excluding:
                print (p, file=f)
        with open("including.lst","w" ) as f:
            for p in include_lst:
                print (p, file=f)
#    include_tree= file_tree(sorted(include_lst))
#    print(include_tree)
    root_dir=include_lst[0].split(os.sep)[0]
    print ("root_dir=",root_dir)
    
    xml= ET.parse(".cproject")
    root=xml.getroot()
#    for child in root:
#        print(child.tag, child.attrib)

    for child in root.iter('sourceEntries'):
        for gchild in child:
            print(gchild.tag)
    
    for child in root.iter('sourceEntries'):
        for gchild in child:
            if gchild.tag == 'entry':
                attr = gchild.attrib
                try:
                    if (attr['kind'] ==  'sourcePath') and (attr['name'] ==  root_dir):
                        child.remove (gchild)
                        print ("Removed existing entry ",gchild.tag)
                        break
                except:
                    print ("error matching attributes for ",gchild.tag)
                    pass
        break #after first   'sourceEntries' - should be just one?      
    ET.SubElement(child, 'entry', {"flags":"VALUE_WORKSPACE_PATH", "kind":"sourcePath", "name":root_dir, "excluding":"|".join(excluding)})
                             
    for child in root.iter('sourceEntries'):
        for gchild in child:
            print(gchild.tag)
            
    oneliner= ET.tostring(root)
    #overwrites original .cproject, may change to somethong different
    with open(".cproject", "wr") as f:
        f.write("""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?>""")
        f.write(oneliner)
            
    print (len(lstFiles), len(lst_a), "last access time = ",latest_at)
                
if __name__ == '__main__':
    proc_tree()
    """
make clean # otherwise compiled files are preserved    
bitbake apps-autoexposure -c cleansstate
bitbake apps-autoexposure -c unpack -f
bitbake apps-autoexposure -c configure -f
./used_files.py sysroots -1
bitbake apps-autoexposure -c compile -f
./used_files.py sysroots 1473297567.42


eyesis@eyesis-SH87R:~/git/elphel393/rootfs-elphel/elphel-apps-autoexposure$ ./used_files.py sysroots/elphel393/usr -1
root_path = sysroots/elphel393/usr
5615 last time =  1473300068.01
eyesis@eyesis-SH87R:~/git/elphel393/rootfs-elphel/elphel-apps-autoexposure$ ./used_files.py sysroots/elphel393/usr 1473300068.01
root_path = sysroots/elphel393/usr
root_dir= sysroots
entry
entry
entry
entry
entry
5615 84 last access time =  1473300080.1
    """