generator for included files, improved partser

parsers/parser_vivado.py

@@ -42,6 +42,7 @@ patternModule= re.compile("module ([A-Za-z_$][A-Za-z_$0-9:]*)\.")
 patternFlatRef=re.compile("(\(([A-Za-z_$][A-Za-z_$0-9]*)\))")
 patternHierRef=re.compile("(\(\\\(\S*) \))")
 patternPin=    re.compile("pin (([A-Za-z_$][A-Za-z_$0-9:]*:[A-Za-z_$][A-Za-z_$0-9:]*(\[\d*])* ))")
+patternHierPin=re.compile("pin ((\\\([A-Za-z_$][A-Za-z_$0-9]*(\[[0-9:]*\])*(\.[A-Za-z_$][A-Za-z_$0-9]*(\[[0-9:]*\])*)* :[A-Za-z_$][A-Za-z_$0-9]*)(\[\d*])* ))")
 
 #PREFIX_REF="@{"
 #SUFFIX_REF="}@"
@@ -69,8 +70,6 @@ global_db={}
 global_pRef=()        
 
 sys.stdout.write("Running: %s %s %s %s\n" % (sys.argv[0],sys.argv[1],sys.argv[2],sys.argv[3]))
-
-
 def isProblem(string):
     if string.startswith("ERROR:") or string.startswith("CRITICAL WARNING:") or string.startswith("WARNING:") or string.startswith("INFO:"):
         return True
@@ -93,12 +92,19 @@ def getLineSignalBit(string):
     if not m:
         m=patternPin.search(string)
         pref="#"
+    if not m:
+        m=patternHierPin.search(string)
+        pref="#"
+            
     if m:    
         line=string[:m.start(1)]+PREFIX_REF+pref+"%s"+SUFFIX_REF+string[m.end(1):]
         ref=m.group(2)
         ref=ref.replace("/",".")
+        ref=ref.replace("\\","") #for patternHierPin only
+        ref=ref.replace(" :",":") #for patternHierPin only
         if "[" in ref:
-            bitStart= ref.find("[")
+#            bitStart= ref.find("[") # aggregate first [index]
+            bitStart= ref.rfind("[") # aggregate last [index]
             bitEnd=ref.find("]",bitStart)
             try:
                 bit=int(ref[bitStart+1:bitEnd])
@@ -137,7 +143,21 @@ def getRanges(pRef):
             h=h+1
         ranges.append((l,h))
     return ranges        
-            
+def rangeToString(ranges): # ranges are sorted in increasing order
+    result=""
+    for (i,rng) in enumerate (reversed(ranges)):
+        if rng[0] >= 0:
+            if i>0:
+                result+=","
+            if (rng[0]<0):
+                return None
+            elif (rng[1]<=(rng[0]+1)):
+                result+="%d"%(rng[0])
+            else:
+                result+="%d:%d"%(rng[1]-1,rng[0])
+#    sys.stdout.write("**** %s ****"%result)
+    return result
+
 def printLineRef(pRef):
     global global_pRef
     global global_mode
@@ -154,13 +174,23 @@ def printLineRef(pRef):
             ref=pRef[1];
             if mod and not ":" in ref:
                 ref= mod+"."+ref
-            for rng in ranges:
-                if (rng[0]<0):
-                    sys.stdout.write(pRef[0]%(ref))
-                elif (rng[1]<=(rng[0]+1)):
-                    sys.stdout.write(pRef[0]%(ref%(rng[0])))
-                else:
-                    sys.stdout.write(pRef[0]%(ref%("%d:%d"%(rng[1]-1,rng[0]))))
+# Replaced with non-Verilog standard ranges, like [11:8,6,4:2,0]                
+#            for rng in ranges:
+#                if (rng[0]<0):
+#                    sys.stdout.write(pRef[0]%(ref))
+#                elif (rng[1]<=(rng[0]+1)):
+#                    sys.stdout.write(pRef[0]%(ref%(rng[0])))
+#                else:
+#                    sys.stdout.write(pRef[0]%(ref%("%d:%d"%(rng[1]-1,rng[0]))))
+
+            srng=rangeToString(ranges);
+            if (srng) :
+                sys.stdout.write(pRef[0]%(ref%(rangeToString(ranges)))) # ("**** [%s] ****"%srng)
+            else:
+                sys.stdout.write(pRef[0]%(ref))   
+#                sys.stdout.write(str(global_mode)+(pRef[0]%(ref)))   
+#           sys.stdout.write(">>>>"+str(ref)+"::::"+str(ranges)+"<<<<") 
+#           sys.stdout.write("global_mode: %d\n" % (global_mode))
             if (global_mode == MODE_ONCE):
                 for rng in ranges:
                     try:
@@ -243,10 +273,10 @@ if isProblem(pline):
         lineHash=hash(outLine)
         if not lineHash in dupLines:
             dupLines.add(lineHash)
-            sys.stdout.write(outline)
+            sys.stdout.write(outLine)
 #            sys.stdout.write(": "+str(lineHash)+" : " +count(dupLines))
     else:
-        sys.stdout.write(outline)    
+        sys.stdout.write(outLine)    
     addTool("",tool)
     
 if global_mode == MODE_POSTPONE:
@@ -255,3 +285,5 @@ if global_mode == MODE_POSTPONE:
             printLineRef((line,ref,0)) # will not add
 #            printLineRef((line,ref,0)) # will not add
 
+sys.stdout.write("Running: %s %s %s %s\n" % (sys.argv[0],sys.argv[1],sys.argv[2],sys.argv[3]))
+sys.stdout.write("global_mode: %d\n" % (global_mode))

src/com/elphel/vdt/VerilogUtils.java

@@ -181,14 +181,30 @@ public class VerilogUtils {
     public static IFile[] getDependencies(IFile [] topFiles) {
     	if (topFiles==null) return null;
         IProject project = topFiles[0].getProject();
-        OutlineDatabase outlineDatabase=getVeditorOutlineDatabase(project);
+        OutlineDatabase outlineDatabase=getVeditorOutlineDatabase(project);
     	return outlineDatabase.getClosureSorted(topFiles);
     } // getDependencies()
     
     public static IFile[] getDependencies(IFile topFile) {
     	return getDependencies(new IFile [] {topFile});
     }    
-/* for now all modules, including library ones */
+
+    /**
+     * Returns included files dependency closure for given verilog file.
+     */
+
+    public static IFile[] getIncludedDependencies(IFile [] topFiles) {
+    	if (topFiles==null) return null;
+        IProject project = topFiles[0].getProject();
+        OutlineDatabase outlineDatabase=getVeditorOutlineDatabase(project);
+    	return outlineDatabase.getCLosureIncludes(topFiles);
+    } // getDependencies()
+    
+    public static IFile[] getIncludedDependencies(IFile topFile) {
+    	return getIncludedDependencies(new IFile [] {topFile});
+    }    
+    
+    /* for now all modules, including library ones */
     
     public static OutlineElement[] getModuleListVeditor(IProject project) {
    		OutlineDatabase database=getVeditorOutlineDatabase(project);

src/com/elphel/vdt/core/launching/VDTLaunchUtil.java

@@ -272,12 +272,20 @@ public class VDTLaunchUtil {
                     dependenciesLocation.add(dependencies[i].getLocation().toOSString());
 //                  System.out.println("  "+dependencies[i].getName());
             }    
-        }
+        }
+        dependencies = VerilogUtils.getIncludedDependencies(file);
+        if(dependencies != null) {
+            for (int i=0; i < dependencies.length; i++) {
+                    dependenciesLocation.add(dependencies[i].getLocation().toOSString());
+//                  System.out.println("  "+dependencies[i].getName());
+            }    
+        }
+        
         return dependenciesLocation;
         
     }
     
-
+//
     /** 
      * Returns an array of environment variables to be used when
      * launching the given configuration or <code>null</code> if unspecified.

src/com/elphel/vdt/core/tools/generators/FilteredIncludesListGenerator.java

+/*******************************************************************************
+ * Copyright (c) 2015 Elphel, Inc.
+ * This file is a part of VDT plug-in.
+ * VDT plug-in is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * VDT plug-in is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ *  Additional permission under GNU GPL version 3 section 7:
+ * If you modify this Program, or any covered work, by linking or combining it
+ * with Eclipse or Eclipse plugins (or a modified version of those libraries),
+ * containing parts covered by the terms of EPL/CPL, the licensors of this
+ * Program grant you additional permission to convey the resulting work.
+ * {Corresponding Source for a non-source form of such a combination shall
+ * include the source code for the parts of Eclipse or Eclipse plugins used
+ * as well as that of the covered work.}
+ *******************************************************************************/
+package com.elphel.vdt.core.tools.generators;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IResource;
+
+
+
+
+
+
+
+
+//import com.elphel.vdt.VDT;
+import com.elphel.vdt.VerilogUtils;
+import com.elphel.vdt.core.tools.params.FormatProcessor;
+import com.elphel.vdt.core.tools.params.Tool;
+import com.elphel.vdt.ui.MessageUI;
+//import com.elphel.vdt.core.verilog.VerilogUtils;
+import com.elphel.vdt.ui.variables.SelectedResourceManager;
+
+/**
+ * Generate the file name list of dependency closure for last selected 
+ * verilog source file. 
+ * 
+ * Created: 21.02.2006
+ * @author  Lvov Konstantin
+ */
+
+public class FilteredIncludesListGenerator extends AbstractGenerator {
+//    public static final String NAME = VDT.GENERATOR_ID_FILTEREDSOURCE_LIST;
+    public static final String NAME = "FilteredIncludesList";
+
+    
+    public FilteredIncludesListGenerator(String prefix, 
+                               String suffix, 
+                               String separator,
+                               FormatProcessor topProcessor) 
+    {
+        super(prefix, suffix, separator, topProcessor ); 
+    }
+    
+    public String getName() {
+        return NAME;
+    }
+
+    protected String[] getStringValues() {
+        String ignoreFilter= SelectedResourceManager.getDefault().getFilter(); // old version
+    	
+//    	System.out.println("FilteredIncludesListGenerator(), tool0="+((tool0==null)?null:(tool0.getName()+tool0.getIgnoreFilter())));
+//    	System.out.print("FilteredIncludesListGenerator(): ");
+    	if (topProcessor!=null){
+    		Tool tool=topProcessor.getCurrentTool();
+//    		System.out.println(", tool="+tool+" tool name="+((tool!=null)?tool.getName():null));
+    		if (tool != null) {
+    			ignoreFilter=tool.getIgnoreFilter();
+//        		System.out.println(" tool="+tool.getName()+", ignoreFilter="+ignoreFilter);
+
+    		} else {
+    			System.out.println("FilteredIncludesListGenerator():  topProcessor.getCurrentTool() is null");
+    		}
+    	} else {
+    		System.out.println("FilteredIncludesListGenerator():  topProcessor is null");
+    	}
+        String[] file_names = null;
+        IResource resource = SelectedResourceManager.getDefault().getChosenVerilogFile();
+        Pattern ignorePattern = null;
+        if (ignoreFilter!=null){
+        	try {
+        		ignorePattern=Pattern.compile(ignoreFilter);
+        	} catch (PatternSyntaxException e){
+        		System.out.println("Error in regular expression for ignore filter: \""+ignoreFilter+"\" - ignoring");
+        		MessageUI.error("Error in regular expression for ignore filter: \""+ignoreFilter+"\" - ignoring");
+        	}
+        }
+        
+        if (resource != null && resource.getType() == IResource.FILE) {
+        	IFile[] files = VerilogUtils.getIncludedDependencies((IFile)resource); // returned just the same x353_1.tf
+        	List<String> fileList=new ArrayList<String>();
+            for (int i=0; i < files.length; i++) {
+            	String fileName=files[i].getProjectRelativePath().toOSString(); //.getName();
+            	if ((ignorePattern!=null) &&ignorePattern.matcher(fileName).matches()) {
+            			continue;
+            	}
+            	fileList.add(fileName);            		
+            }
+            file_names=fileList.toArray(new String[0]);
+        } else {
+//            fault("There is no selected project");
+            System.out.println(getName()+": no project selected");
+            return new String[] {""};
+        }
+        return file_names;
+    }
+
+} // class FilteredIncludesListGenerator

src/com/elphel/vdt/core/tools/generators/IncludesListGenerator.java

+/*******************************************************************************
+ * Copyright (c) 2015 Elphel, Inc.
+ * Copyright (c) 2006 Elphel, Inc and Excelsior, LLC.
+ * This file is a part of VDT plug-in.
+ * VDT plug-in is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * VDT plug-in is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ *  Additional permission under GNU GPL version 3 section 7:
+ * If you modify this Program, or any covered work, by linking or combining it
+ * with Eclipse or Eclipse plugins (or a modified version of those libraries),
+ * containing parts covered by the terms of EPL/CPL, the licensors of this
+ * Program grant you additional permission to convey the resulting work.
+ * {Corresponding Source for a non-source form of such a combination shall
+ * include the source code for the parts of Eclipse or Eclipse plugins used
+ * as well as that of the covered work.}
+ *******************************************************************************/
+package com.elphel.vdt.core.tools.generators;
+
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IResource;
+
+
+//import com.elphel.vdt.VDT;
+import com.elphel.vdt.VerilogUtils;
+//import com.elphel.vdt.core.verilog.VerilogUtils;
+import com.elphel.vdt.ui.variables.SelectedResourceManager;
+
+/**
+ * Generate the file name list of dependency closure for last selected 
+ * verilog source file. 
+ * 
+ * Created: 21.02.2006
+ * @author  Lvov Konstantin
+ */
+
+public class IncludesListGenerator extends AbstractGenerator {
+//    public static final String NAME = VDT.GENERATOR_ID_SOURCE_LIST;
+    public static final String NAME = "IncludesList";
+
+    
+    public IncludesListGenerator(String prefix, 
+                               String suffix, 
+                               String separator) 
+    {
+        super(prefix, suffix, separator,null);  // null for topFormatProcessor - this generator can not reference other parameters
+    }
+    
+    public String getName() {
+        return NAME;
+    }
+
+    protected String[] getStringValues() {
+        String[] file_names = null;
+//        IResource resource = SelectedResourceManager.getDefault().getSelectedVerilogFile();
+        IResource resource = SelectedResourceManager.getDefault().getChosenVerilogFile();
+        if (resource != null && resource.getType() == IResource.FILE) {
+        	IFile[] files = VerilogUtils.getDependencies((IFile)resource); // returned just the same x353_1.tf
+            file_names = new String[files.length];
+            for (int i=0; i < files.length; i++)
+                file_names[i] = files[i].getProjectRelativePath().toOSString(); //.getName();
+        } else {
+//            fault("There is no selected project");
+            System.out.println(getName()+": no project selected");
+            return new String[] {""};
+        }
+        return file_names;
+    }
+
+} // class IncludesListGenerator

src/com/elphel/vdt/core/tools/params/recognizers/DefaultListGeneratorRecognizer.java

@@ -29,7 +29,7 @@ package com.elphel.vdt.core.tools.params.recognizers;
 import com.elphel.vdt.core.Utils;
 import com.elphel.vdt.core.tools.generators.*;
 import com.elphel.vdt.core.tools.params.FormatProcessor;
-import com.elphel.vdt.core.tools.params.Tool;
+//import com.elphel.vdt.core.tools.params.Tool;
 
 
 public class DefaultListGeneratorRecognizer implements Recognizer {
@@ -42,6 +42,8 @@ public class DefaultListGeneratorRecognizer implements Recognizer {
     	AbstractGenerator[] generators=  new AbstractGenerator[]{
     			new SourceListGenerator("","",null),
     			new FilteredSourceListGenerator("","",null, topProcessor),
+    			new IncludesListGenerator("","",null),
+    			new FilteredIncludesListGenerator("","",null, topProcessor),
     			new FileListGenerator("","",null)
     	};
     	this.generators=generators;

src/com/elphel/vdt/core/tools/params/recognizers/ParamRepeaterRecognizer.java

@@ -30,6 +30,8 @@ import com.elphel.vdt.core.tools.generators.AbstractGenerator;
 import com.elphel.vdt.core.tools.generators.FileListGenerator;
 import com.elphel.vdt.core.tools.generators.FilteredSourceListGenerator;
 import com.elphel.vdt.core.tools.generators.SourceListGenerator;
+import com.elphel.vdt.core.tools.generators.FilteredIncludesListGenerator;
+import com.elphel.vdt.core.tools.generators.IncludesListGenerator;
 import com.elphel.vdt.core.tools.generators.TopModulesNameGenerator;
 import com.elphel.vdt.core.tools.generators.ValueGenerator;
 import com.elphel.vdt.core.tools.params.FormatProcessor;
@@ -72,6 +74,10 @@ public class ParamRepeaterRecognizer extends RepeaterRecognizer {
             return new FilteredSourceListGenerator(repPrefix, repSuffix, separator, topProcessor);
         else if(genName.equals(SourceListGenerator.NAME))
             return new SourceListGenerator(repPrefix, repSuffix, separator);
+        if(genName.equals(FilteredIncludesListGenerator.NAME))
+            return new FilteredIncludesListGenerator(repPrefix, repSuffix, separator, topProcessor);
+        else if(genName.equals(IncludesListGenerator.NAME))
+            return new IncludesListGenerator(repPrefix, repSuffix, separator);
         else if(genName.equals(FileListGenerator.NAME))
             return new FileListGenerator(repPrefix, repSuffix, separator);
         else if(genName.equals(TopModulesNameGenerator.NAME))

src/com/elphel/vdt/core/tools/params/recognizers/RepeaterRecognizer.java

@@ -31,7 +31,9 @@ import com.elphel.vdt.core.tools.params.FormatProcessor;
 import com.elphel.vdt.core.tools.params.ToolException;
 import com.elphel.vdt.core.tools.generators.AbstractGenerator;
 import com.elphel.vdt.core.tools.generators.FileListGenerator;
-import com.elphel.vdt.core.tools.generators.FilteredSourceListGenerator;
+import com.elphel.vdt.core.tools.generators.FilteredSourceListGenerator;
+import com.elphel.vdt.core.tools.generators.FilteredIncludesListGenerator;
+import com.elphel.vdt.core.tools.generators.IncludesListGenerator;
 import com.elphel.vdt.core.tools.generators.TopModulesNameGenerator;
 import com.elphel.vdt.core.tools.generators.SourceListGenerator;
 import com.elphel.vdt.ui.MessageUI;
@@ -161,6 +163,10 @@ public class RepeaterRecognizer implements Recognizer {
     	if (genName.equals(gen.getName())) return gen;
     	gen=new SourceListGenerator(repPrefix, repSuffix, separator);
     	if (genName.equals(gen.getName())) return gen;
+    	gen=new FilteredIncludesListGenerator(repPrefix, repSuffix, separator,topProcessor);
+    	if (genName.equals(gen.getName())) return gen;
+    	gen=new IncludesListGenerator(repPrefix, repSuffix, separator);
+    	if (genName.equals(gen.getName())) return gen;
     	gen=new FileListGenerator(repPrefix, repSuffix, separator);
     	if (genName.equals(gen.getName())) return gen;
     	gen=new TopModulesNameGenerator(repPrefix, repSuffix, separator);