Commit e32fcf20 authored by Andrey Filippov's avatar Andrey Filippov

modified run_bitbake.sh to be run from subprojects, re-wrote used_files.py

parent 4d42bcfe
...@@ -8,3 +8,6 @@ rootfs-elphel ...@@ -8,3 +8,6 @@ rootfs-elphel
tools tools
bootable-images bootable-images
projects.json projects.json
.project
.pydevproject
.settings
#!/bin/bash #!/bin/bash
#ls -all
args="$@" args="$@"
arg0=$0
#echo "arg0=" $arg0
#echo "args=" $args
while (( "$#" )); do while (( "$#" )); do
shift shift
done done
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" #DIR0="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DIR=$(dirname $(dirname $(dirname $(realpath $(dirname $(dirname $arg0))/sysroots))))
echo "DIR=" $DIR
echo "Launching bitbake $args" echo "Launching bitbake $args"
cd $DIR/../../poky #cd $DIR0/../../poky
cd $DIR
. ./oe-init-build-env . ./oe-init-build-env
bitbake $args | sed -u 's@| @@' bitbake $args | sed -u 's@| @@'
exit 0 exit 0
#!/usr/bin/env python #!/usr/bin/env python
# encoding: utf-8 # encoding: utf-8
from __future__ import division
from __future__ import print_function from __future__ import print_function
""" from __future__ import division
'''
# @file used_files.py # @file used_files.py
# @brief Extract file access data after build, modify CDT project configuration # @brief Extract file access data after build, modify CDT project configuration
# (.cproject) accordingly # (.cproject) accordingly
...@@ -20,52 +21,35 @@ from __future__ import print_function ...@@ -20,52 +21,35 @@ from __future__ import print_function
# along with this program. If not, see <http:#www.gnu.org/licenses/>. # along with this program. If not, see <http:#www.gnu.org/licenses/>.
@author: Andrey Filippov @author: Andrey Filippov
@copyright: 2015 Elphel, Inc.
@license: GPLv3.0+ @license: GPLv3.0+
@contact: andrey@elphel.coml @contact: andrey@elphel.coml
@deffield updated: Updated @deffield updated: Updated
""" '''
__author__ = "Andrey Filippov" __author__ = "Andrey Filippov"
__copyright__ = "Copyright 2016, Elphel, Inc." __copyright__ = "Copyright 2015, Elphel, Inc."
__license__ = "GPL" __license__ = "GPL"
__version__ = "3.0+" __version__ = "3.0+"
__maintainer__ = "Andrey Filippov" __maintainer__ = "Andrey Filippov"
__email__ = "andrey@elphel.com" __email__ = "andrey@elphel.com"
__status__ = "Development" __status__ = "Development"
import sys
import os import os
import sys
import time import time
import xml.etree.ElementTree as ET import subprocess
''' import datetime
TODO:Automate, find out why separate touch commands are needed import xml.etree.ElementTree
Run this program twice: MAIN_SRC = 'src' # main source subdirectory path relative to project directory. SHould not be './' !
1-st run ./used_files.py linux -1 # print (" cwd=",os.getcwd())
and save shown timestamp def get_bitbake_target(project_root):
Then run (some mystery here) cproject_path = os.path.join(project_root,".cproject")
touch src/drivers/ata/ahci_elphel.c try:
touch src/drivers/elphel/sensor_common.c return xml.etree.ElementTree.parse(cproject_path).getroot().find(
Wait 5 seconds and run (in a different console with appropriate sourcing) 'storageModule/cconfiguration/storageModule/configuration/folderInfo/toolChain/builder').get('arguments').split()[0]
bitbake linux-xlnx -c compile -f except:
Then again return None
./used_files.py linux <timestamp_from_the_first_run>
If somethong went wrong you will need to resore .cproject from eclipse_project_setup directory
For php:
bitbake php -c cleansstate
bitbake php -c unpack -f
bitbake php -c configure -f
./used_files.py php -1
bitbake php -c compile -f
./used_files.py php 1471044836.8
Doing:
bitbake php -c compile -f
./used_files.py php -1
bitbake php -c compile -f
./used_files.py php 1471044836.8
Corrupts make process
'''
def file_tree(flist): # Each file in list is a file, no directories def file_tree(flist): # Each file in list is a file, no directories
ftree={} ftree={}
for p in flist: for p in flist:
...@@ -120,20 +104,66 @@ def exclude_list(ftree, flist): ...@@ -120,20 +104,66 @@ def exclude_list(ftree, flist):
pl = sorted (pl) pl = sorted (pl)
return pl return pl
def get_sourceEntries(xml_root, root_dir):
def proc_tree(): # main_src = 'src' # main source folder
DEBUG = True for sm in xml_root.iter('storageModule'):
extensions = [".h",".c",".cpp"] attr = sm.attrib
exclude_start = ["linux"+os.sep+"scripts"+os.sep,"linux"+os.sep+"source"+os.sep+"scripts"+os.sep]
delta_t = 3 # seconds
try: try:
root_path = sys.argv[1] if sm.attrib['moduleId'] != 'cdtBuildSystem':
continue
except: except:
print ("Calling %s <root directory path> [timestamp]"%(os.path.basename(sys.argv[0]))) continue
for se in sm.iter('sourceEntries'):
for en in se:
if en.tag == 'entry':
attr = en.attrib
try: try:
start_time = float(sys.argv[2]) if (attr['kind'] == 'sourcePath') and (attr['name'] == root_dir):
se.remove (en)
print ("Removed existing entry, name= ",attr['name'])
return se
except:
print ("error matching attributes for ",en.tag)
pass
#look for MAIN_SRC entry
for se in sm.iter('sourceEntries'):
for en in se:
if en.tag == 'entry':
attr = en.attrib
try:
if (attr['kind'] == 'sourcePath') and (attr['name'] == MAIN_SRC):
print ("Found existing entry for main source folder, name= ",attr['name'])
return se
except:
print ("error matching attributes for ",en.tag)
pass
#create new sourceEntries
print ("Creating new sourceEntries element")
try:
se = xml.etree.ElementTree.SubElement(sm.find('configuration'), 'sourceEntries')
#first entry - src folder
xml.etree.ElementTree.SubElement(se, 'entry', {"flags":"VALUE_WORKSPACE_PATH", "kind":"sourcePath", "name":MAIN_SRC})
return se
except: except:
start_time = 0.0 return None
return None
def proc_tree(root_path, start_time, output_project_file, DEBUG): # string, float
print("root_path=",root_path)
print("start_time=",start_time)
print("output_project_file=",output_project_file)
print("DEBUG=",DEBUG)
extensions = [".h",".c",".cpp"]
exclude_start = ["linux"+os.sep+"scripts"+os.sep,"linux"+os.sep+"source"+os.sep+"scripts"+os.sep]
delta_t = 3 # seconds
# try:
# start_time = float(sys.argv[2])
# except:
# start_time = 0.0
touch_files= start_time < 0.0 touch_files= start_time < 0.0
print ("root_path = %s"%(root_path)) print ("root_path = %s"%(root_path))
...@@ -188,12 +218,16 @@ def proc_tree(): ...@@ -188,12 +218,16 @@ def proc_tree():
include_lst.append(p) include_lst.append(p)
if touch_files: if touch_files:
print (len(lstFiles), "last time = ", time.time()) print (len(lstFiles), "last time = ", time.time())
return return time.time()
excluding = exclude_list(all_tree, include_lst) excluding = exclude_list(all_tree, include_lst)
# print (all_tree) # print (all_tree)
# print (sorted(include_lst)) # print (sorted(include_lst))
# print ("|".join(excluding)) # print ("|".join(excluding))
#os.sep.join(s1.split(os.sep)[1:])
including=[]
#get rid of top directory in include paths
for p in include_lst:
including.append(os.sep.join(p.split(os.sep)[1:]))
if DEBUG: if DEBUG:
with open("all_sources.lst","w" ) as f: with open("all_sources.lst","w" ) as f:
for p in sorted(lstFiles): for p in sorted(lstFiles):
...@@ -213,72 +247,154 @@ def proc_tree(): ...@@ -213,72 +247,154 @@ def proc_tree():
for p in excluding: for p in excluding:
print (p, file=f) print (p, file=f)
with open("including.lst","w" ) as f: with open("including.lst","w" ) as f:
for p in include_lst: for p in including:
print (p, file=f) print (p, file=f)
# include_tree= file_tree(sorted(include_lst)) # include_tree= file_tree(sorted(include_lst))
# print(include_tree) # print(include_tree)
root_dir=include_lst[0].split(os.sep)[0] try:
root_dir=include_lst[0].split(os.sep)[0] #may fail if list is empty
print ("root_dir=",root_dir) print ("root_dir=",root_dir)
except:
print ("No files used from ",root_path)
root_dir=root_path
xml= ET.parse(".cproject") root= xml.etree.ElementTree.parse(".cproject").getroot()
root=xml.getroot()
# for child in root:
# print(child.tag, child.attrib)
for child in root.iter('sourceEntries'): if len(include_lst):
for gchild in child: se = get_sourceEntries(root, root_dir)
print(gchild.tag) if se is None:
print ("No sourceEntries exist and could not create one")
return -1
#add other header files in header directory, excluding...
# if len(excluding) < len(including):
xml.etree.ElementTree.SubElement(se,
'entry',
{"flags":"VALUE_WORKSPACE_PATH", "kind":"sourcePath", "name":root_dir, "excluding":"|".join(excluding)})
# else:
# xml.etree.ElementTree.SubElement(se,
# 'entry',
# {"flags":"VALUE_WORKSPACE_PATH", "kind":"sourcePath", "name":root_dir, "including":"|".join(including)})
for child in root.iter('sourceEntries'):
for gchild in child:
if gchild.tag == 'entry':
attr = gchild.attrib
try:
if (attr['kind'] == 'sourcePath') and (attr['name'] == root_dir):
child.remove (gchild)
print ("Removed existing entry ",gchild.tag)
break
except:
print ("error matching attributes for ",gchild.tag)
pass
break #after first 'sourceEntries' - should be just one?
ET.SubElement(child, 'entry', {"flags":"VALUE_WORKSPACE_PATH", "kind":"sourcePath", "name":root_dir, "excluding":"|".join(excluding)})
for child in root.iter('sourceEntries'): for child in root.iter('sourceEntries'):
for gchild in child: for gchild in child:
print(gchild.tag) print("tag=",gchild.tag," name=",gchild.attrib['name']," kind=",gchild.attrib['kind'])
oneliner= xml.etree.ElementTree.tostring(root)
oneliner= ET.tostring(root)
#overwrites original .cproject, may change to somethong different #overwrites original .cproject, may change to somethong different
with open(".cproject", "wr") as f: with open(output_project_file, "wr") as f:
f.write("""<?xml version="1.0" encoding="UTF-8" standalone="no"?> f.write("""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?>""") <?fileVersion 4.0.0?>""")
f.write(oneliner) f.write(oneliner)
print (len(lstFiles), len(lst_a), "last access time = ",latest_at) print (len(lstFiles), len(lst_a), "last access time = ",latest_at)
return latest_at
def main():
DEBUG = False
output_project_file=".cproject" #".cproject_new" Overwrite or save to a new file
argv = sys.argv
top_h_dir='sysroots'
if len(argv) < 2:
print ("""
This program creates a list of the source files (such as C headers) used to build the project
for Eclipse CDT plugin. It only needs to be run for the new projects, the existing ones already
have .cproject file distributed in each subproject repository (in 'eclipse_project_setup' sub-
directory of the root sub-project directory). This sub-directory content is copyied to the sub-
project root by setup.py when it is first run. You may delete .project file (while Eclipse is closed)
and re-run setup.py (in elphel393 directory) restore default project settings if they get corrupted.
The program does not rely on exact duplicating of the environment used by bitbake, instead it
"spies" on the bitbake by noticing which files it accesses during build (using file modification and
access timestamps) and creates list of exclusions. As the access time is recorded only after the first
access after modification, the program first unpacks/touches the sources.
The extra_source directory is the specified in the command line directory (in addition to hard-coded
'src') with sub-tree of the source files (headers) to be filtered.
Here is the full sequence (target is the project name extracted from .cproject):
1. bitbake target-c cleansstate
2. bitbake target-c unpack -f
3. bitbake target-c configure -f
4. Scan all files under extra_source directory, find last modification stamp
5. bitbake target-c compile -f
6. bitbake target-c install -f # in the case of Linux kernel in triggers compilation of the kernel
modules
7. Scan source files, create list of the used files and then list of excluded files (Eclipse CDT allows
to specify directory and exclusion filter)
8. Add (or replace) the record in .cproject file that specifies source directory and the filter
You need to run indexing (right-click on the project in the Navigator panel -> Index -> rebuild
when the workspace is opened with the modified .cproject file.
Do not run this program when Eclipse IDE is opened that includes the current project!
The program should be launched from the project root directory, e.g. for applications:
./scripts/used_files.py sysroots
Program can either overwrite the current .cproject configuration file or create a new modified version
if specified in teh command line argument. In that case program runs in debug mode and generates lists
of files in the project root directory:
all_sources.lst - all scanned source file with last access timestamps
including.lst - list of the files (relative to specified extra_source directory) used by bitbake
excluding.lst - list of the unused files (they will appear crossed in the Project Navigator)
USAGE:
%s extra_source [path-to-modified-cproject]
First (mandatory) argument of this program (extra_source) is the relative path additional source/header
files. For Linux kernel development it is 'linux', for php extension - 'php, for applications - 'sysroots'
(symlink to header files)
Second (optional) argument (path-to-modified-cproject) is the relative to project root file to write
modified .cproject content. If specified it forces program to run in debug mode and generate 3 file lists.
"""%(argv[0],))
return 0
# Check that there is MAIN_SRC ('src') subdirectory in the project directory
if not os.path.isdir(MAIN_SRC):
print("\n*** Project source files should be in subdirectory '%s' for this program to run. ***\n"%(MAIN_SRC,))
return 1
top_h_dir = argv[1]
if len(argv) > 2:
output_project_file = argv[2] # Save result .cproject to a new file
DEBUG = True
print (" cwd=",os.getcwd())
bitbake_target = get_bitbake_target(os.getcwd())
print ("bitbake target=",bitbake_target)
if not bitbake_target:
print ("Failed to find bitbake target from .cproject file (it has to be set up in project->properties->C/C++ Build command")
print ("For example: ${workspace_loc:/linux-elphel/scripts/run_bitbake.sh} linux-xlnx")
return 1
bitbake = './scripts/run_bitbake.sh'
cmnd = bitbake+ ' '+bitbake_target+' -c cleansstate'
# subprocess.call("ls -all", shell = True)
return_code = subprocess.call(cmnd, shell = True)
print ('Command: %s returned %d'%(cmnd,return_code))
cmnd = bitbake+ ' '+bitbake_target+' -c unpack -f'
return_code = subprocess.call(cmnd, shell = True)
print ('Command: %s returned %d'%(cmnd,return_code))
cmnd = bitbake+ ' '+bitbake_target+' -c configure -f'
return_code = subprocess.call(cmnd, shell = True)
print ('Command: %s returned %d'%(cmnd,return_code))
last_mod = proc_tree(top_h_dir, -1.0, output_project_file, DEBUG);
print ('last_mod=',last_mod)
print("waiting 5 seconds")
time.sleep(5)
print("waiting over")
cmnd = bitbake+ ' '+bitbake_target+' -c compile -f'
return_code = subprocess.call(cmnd, shell = True)
print ('Command: %s returned %d'%(cmnd,return_code))
cmnd = bitbake+ ' '+bitbake_target+' -c install -f'
return_code = subprocess.call(cmnd, shell = True)
print ('Command: %s returned %d'%(cmnd,return_code))
latest_at = proc_tree(top_h_dir, last_mod+3, output_project_file, DEBUG);
print ('latest_at=',latest_at)
return 0
if __name__ == '__main__': if __name__ == "__main__":
proc_tree() sys.exit(main())
""" \ No newline at end of file
make clean # otherwise compiled files are preserved
bitbake apps-autoexposure -c cleansstate
bitbake apps-autoexposure -c unpack -f
bitbake apps-autoexposure -c configure -f
./used_files.py sysroots -1
bitbake apps-autoexposure -c compile -f
./used_files.py sysroots 1473297567.42
eyesis@eyesis-SH87R:~/git/elphel393/rootfs-elphel/elphel-apps-autoexposure$ ./used_files.py sysroots/elphel393/usr -1
root_path = sysroots/elphel393/usr
5615 last time = 1473300068.01
eyesis@eyesis-SH87R:~/git/elphel393/rootfs-elphel/elphel-apps-autoexposure$ ./used_files.py sysroots/elphel393/usr 1473300068.01
root_path = sysroots/elphel393/usr
root_dir= sysroots
entry
entry
entry
entry
entry
5615 84 last access time = 1473300080.1
"""
\ No newline at end of file
...@@ -46,7 +46,7 @@ def main(): ...@@ -46,7 +46,7 @@ def main():
should have CDT and EGit plugins installed. should have CDT and EGit plugins installed.
USAGE: USAGE:
%s eclipse-home [[path-to-workspace] project-paths] %s eclipse-home [[path-to-workspace] project-paths]
First (mandatory) argument of this program (eclipse-home) is the full path to Eclipse instalaltion First (mandatory) argument of this program (eclipse-home) is the full path to Eclipse installation
(directory that has eclipse executable and eclipse.ini files). (directory that has eclipse executable and eclipse.ini files).
Second (optional) argument (path-to-workspace) is the path to workspace. If not specified, Second (optional) argument (path-to-workspace) is the path to workspace. If not specified,
then ../workspace_elphel393 will be used. then ../workspace_elphel393 will be used.
...@@ -55,7 +55,7 @@ USAGE: ...@@ -55,7 +55,7 @@ USAGE:
The program will not overwrite or modify any existing workspace. The program will not overwrite or modify any existing workspace.
"""%(argv[0],)) """%(argv[0],))
return return 0
eclipse_home = argv[1] eclipse_home = argv[1]
if (len(argv) > 3): if (len(argv) > 3):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment