Minor changes while better integrating cocotb

parsers/parser_cocotb_icarus.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013, Elphel.inc.
+# configuration of the DDR-related registers
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#  
+#  Additional permission under GNU GPL version 3 section 7:
+#  If you modify this Program, or any covered work, by linking or combining it
+#  with Eclipse or Eclipse plugins (or a modified version of those libraries),
+#  containing parts covered by the terms of EPL/CPL, the licensors of this
+#  Program grant you additional permission to convey the resulting work.
+#  {Corresponding Source for a non-source form of such a combination shall
+#  include the source code for the parts of Eclipse or Eclipse plugins used
+#  as well as that of the covered work.}
+ 
+__author__ = "Andrey Filippov"
+__copyright__ = "Copyright 2016, Elphel, Inc."
+__license__ = "GPL"
+__version__ = "3.0+"
+__maintainer__ = "Andrey Filippov"
+__email__ = "andrey@elphel.com"
+__status__ = "Development"
+"""
+Mimics cocotb/Python problems to those of Icarus, passing icarus through
+<Filepath>:<line>: [warning|error|info]: <description>
+"""
+
+
+import sys
+import re
+RAISED_EXCEPTION="raised exception:"
+COMMA_LINE=", line"
+state = None # None - pass, cocotb - collecting Cocotb problem, python- just raw python problem
+lines = []
+def report_python(lines):
+    err_msg=lines[0]
+    for i in range((len(lines)-1)//2):
+        fline=lines[2*i+1]
+        fileStart= fline.find('"')+1
+        fileEnd=   fline.find('"',fileStart)
+        fpath = fline[fileStart:fileEnd]
+        flineNumStart=fline.find(COMMA_LINE,fileEnd)+len(COMMA_LINE)
+        flineNumEnd=  fline.find(',',flineNumStart)
+        flineNum = fline[flineNumStart:flineNumEnd].strip()
+        infunction = fline[flineNumEnd+1:].strip()
+        lineTxt = lines[2*i+2]
+        sys.stdout.write('%s:%s: error: %s, %s "%s"\n'%(fpath, flineNum, err_msg, infunction, lineTxt))
+
+for line in iter(sys.stdin.readline,''):
+    sline = line.strip()
+    if state == "cocotb":
+        if (len(lines)%2 == 0) or sline.startswith('File "'):
+            lines.append(sline)
+        else:
+            report_python(lines)
+            state = None
+            sys.stdout.write(line)    
+    elif state == "python":
+        if (len(lines)%2 == 0) or sline.startswith('File "'):
+            lines.append(sline)
+        else:
+            lines[0] = sline # instead of "Traceback (most recent call last):"
+            report_python(lines)
+            state = None
+    else:
+        if sline.startswith("Traceback (most"):
+            #print("***Got Traceback***")
+            state = "python"
+            lines =[sline]
+            continue
+        elif (line.find("ERROR")>=0) and (line.find(RAISED_EXCEPTION)>0):
+            #print("***Got Cocotb error***")
+            index=line.find(RAISED_EXCEPTION)+len(RAISED_EXCEPTION)
+            sline=  line[index:].strip()
+            state = "cocotb"
+            lines =[sline]
+            sys.stdout.write(line)
+            
+            
+            
+            continue
+        else:
+            sys.stdout.write(line)    
+            
+    
+        
+"""        
+'
+    if isProblem(pline):
+        if line.startswith("   ") :
+            pline = pline[:len(pline)-1]+line[2:]
+        else:
+            pline=addTool(pline,tool)
+#            sys.stdout.write("*"+str(debugSize())+pline)
+#            sys.stdout.write(pline)
+            if REMOVE_DUPS:
+                lineHash=hash(pline)
+                if not lineHash in dupLines:
+                    dupLines.add(lineHash)
+                    sys.stdout.write(pline)
+#                    sys.stdout.write(": "+str(lineHash)+" : " +count(dupLines))
+            else:
+                sys.stdout.write(pline)    
+            
+            
+            pline = line
+    else:
+        pline = line
+"""
\ No newline at end of file

src/com/elphel/vdt/core/launching/RunningBuilds.java

@@ -314,9 +314,13 @@ public class RunningBuilds {
 					if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_LAUNCHING)) System.out.print("Killed open console");
 					return true;
 				} else {
+					if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_LAUNCHING)){
+						System.out.print("removeConsole(): hasBad="+hasBad+", hasGood="+hasGood+", goodSet"+goodSet);
+					}
 					if      (hasBad)  tool.setState(TOOL_STATE.FAILURE);
 					else if (hasGood) tool.setState(TOOL_STATE.SUCCESS);
 					else if (goodSet) tool.setState(TOOL_STATE.FAILURE);
+					else              tool.setState(TOOL_STATE.SUCCESS); // No bad, and good is not specified
 				}
 			}
 		}

src/com/elphel/vdt/core/tools/config/xml/ParamNodeReader.java

@@ -49,10 +49,12 @@ public class ParamNodeReader extends AbstractConditionNodeReader {
             try {
             	Parameter param = readParam(node, condition);
             	String id=param.getID();
+            	// It should now handle duplicate parameters (used for conditionals) AF 2016/07/04
+            	/*
             	if (paramIdList.contains(id)){
             		System.out.println("Warning: duplicate parameter ('" + id + "') in context '" + context + "' defined in "+param.getSourceXML());
-//                    throw new ConfigException("Duplicate parameter ('" + id + "') in context '" + context + "' defined in "+param.getSourceXML());
             	}
+            	*/
                 paramIdList.add(id);
                 paramList.add(param);
             } catch(ConfigException e) {

src/com/elphel/vdt/core/tools/generators/AbstractGenerator.java

@@ -155,7 +155,8 @@ public abstract class AbstractGenerator {
     protected String fault(String message) {
     	if (menuMode)
     		return "";
-        MessageUI.error("Generator '" + getName() + "' fault: " + message);
+//        MessageUI.error("Generator '" + getName() + "' fault: " + message);
+        System.out.println("Error: Generator '" + getName() + "' fault: " + message);
         return null;
     }
     

src/com/elphel/vdt/ui/options/ContextOptionsDialog.java

@@ -43,6 +43,7 @@ import com.elphel.vdt.veditor.preference.PreferenceStrings;
 import com.elphel.vdt.core.options.OptionsCore;
 import com.elphel.vdt.core.tools.contexts.Context;
 import com.elphel.vdt.core.tools.contexts.PackageContext;
+import com.elphel.vdt.core.tools.params.Parameter;
 import com.elphel.vdt.core.tools.params.ToolException;
 import com.elphel.vdt.core.tools.params.ToolSequence;
 import com.elphel.vdt.ui.MessageUI;
@@ -88,7 +89,7 @@ public class ContextOptionsDialog extends Dialog {
     protected void okPressed() {
     	/* Currently multiple same-named parameters in the same context have warnings.
     	 * What happens is that after parameters are changed in the dialog, the new value is
-    	 * applied only to the first entry, then the saecond (unmodified) entry is processed
+    	 * applied only to the first entry, then the second (unmodified) entry is processed
     	 * and overwrites the modified value (effectively disabling modification. Another
     	 * option would be to partially restore name+index in the preference store - just
     	 * instead of the index of all parameters, use index among those with the same ID.
@@ -98,6 +99,11 @@ public class ContextOptionsDialog extends Dialog {
     	 * tool settings */
     	
     	optionsBlock.performApply();
+    	//Debug:
+//    	for (Parameter par:context.getParams()){
+//    		System.out.println("okPressed() "+context.getName()+" ->"+par.getID());
+//    	}
+    	
         OptionsCore.doStoreContextOptions(context, store);
         context.setWorkingDirectory(location);
         try {

src/com/elphel/vdt/ui/options/OptionsBlock.java

@@ -99,7 +99,16 @@ public class OptionsBlock {
         Iterator<Entry<Parameter, Component>> i = components.entrySet().iterator(); 
         while (i.hasNext()) {
             Component component = i.next().getValue();
+            List<Parameter> pl =  component.getSameParameters(); 
             component.performApply();
+            if (pl.size() > 0 ) {
+//            	System.out.println("performApply(): component->param->id="+component.getParam().getID()
+//            			+" has "+pl.size()+" extra parameter(s) in the same context:");
+            	for (Parameter param:pl){
+            		component.duplicateParamValue(param);
+            	}
+            	
+            }
         }
 	}
 

src/com/elphel/vdt/ui/options/component/Component.java

@@ -26,6 +26,9 @@
  *******************************************************************************/
 package com.elphel.vdt.ui.options.component;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.swt.SWT;
 import org.eclipse.swt.layout.GridData;
@@ -43,6 +46,7 @@ import org.eclipse.swt.graphics.Color;
 //import com.elphel.vdt.VDTPlugin;
 import com.elphel.vdt.veditor.VerilogPlugin;
 import com.elphel.vdt.core.tools.params.Parameter;
+import com.elphel.vdt.core.tools.params.ToolException;
 
 public abstract class Component {
 
@@ -68,6 +72,14 @@ public abstract class Component {
 
     protected ChangeListener changeListener;
     
+    // To be able to apply the same value to all parameters having the same name in the same context
+    public List<Parameter> getSameParameters(){
+    	List<Parameter> lp = new ArrayList<Parameter>();
+    	for (Parameter par: param.getContext().getParams()){
+    		if ((param.getID().equals(par.getID())) && (param!=par) ) lp.add(par);
+    	}
+    	return lp;
+    }
     public Component(Parameter param) {
         this(param, null);
     }
@@ -132,6 +144,25 @@ public abstract class Component {
     
     public abstract String performApply();
 
+    public void duplicateParamValue(Parameter param){
+    	if (isDisposed()) return;
+    	if (isDefault){
+    		param.setToDefault();
+    	} else if (param.getType().isList()){
+    		try {
+				param.setCurrentValue(this.param.getCurrentValue());
+			} catch (ToolException e) {
+				System.out.println("duplicateParamValue(): failed to set List value for "+param.getID()); 
+			}
+    	} else {
+			try {
+				param.setCurrentValue(this.param.getCurrentValue().get(0));
+			} catch (ToolException e) {
+				System.out.println("duplicateParamValue(): failed to set String value for "+param.getID()); 
+			}
+    	}
+    }
+
     public abstract void setPreferenceStore(IPreferenceStore store);
 
     protected abstract void saveControlState();

src/com/elphel/vdt/ui/views/DesignFlowView.java

@@ -1007,6 +1007,8 @@ public class DesignFlowView extends ViewPart implements ISelectionListener {
     		int choice,
     		String fullPath,
     		String ignoreFilter) throws CoreException {
+    	// Will delete conditionally (controlled by preferences)
+		VerilogPlugin.deleteExternalMarkers(selectedResource.getProject());
     	if (tool != null) {
     		toolSequence.launchToolSequence(tool,mode, choice, fullPath, ignoreFilter);
     	} else if (selectedItem.hasChildren()) {

tools/Verilog/Cocotb.xml

@@ -115,9 +115,6 @@
 		shell="bash"
 		package="FPGA_package"
 		interface="Cocotb"
-		errors="(.*):([0-9]+): [a-z_\- ]*error[: ]?(.*)"
-		warnings="(.*):([0-9]+): [a-z_\- ]*[warning|sorry]: (.*)"
-		info="\s*File\s+"(.*)", line ([0-9]+)[, in ]?(.*)"
 		top-file="%CocotbDutTopFile"
 		define="COCOTB"
 		>
@@ -265,6 +262,12 @@
 		<parameter id="ShowWarnings" type="BoolYesNo" format="None"
 			default="true" label="Show output warnings" />
 
+		<parameter id="ShowInfo" type="BoolYesNo" format="None"
+			default="true" label="Show info messages" />
+
+		<parameter id="FilterParser" type="BoolYesNo" format="None"
+			default="true" label="Filter simulation output with an external parser" />
+
 		<parameter id="RemoveBugs" type="BoolYesNo" format="None"
 			default="false" label="Remove buggy simulator output" />
 
@@ -288,15 +291,6 @@
 			type="String" default="%%CurrentFileBase" format="FstFileSyntax"
 			readonly="false" />
 
-
-		<parameter id="GrepFindErr" label="Grep pattern for errors only"
-			type="String" format="GrepFindSyntax" default="error" readonly="false"
-			visible="true" />
-
-		<parameter id="GrepFindErrWarn" label="Grep pattern for both errors and warnings"
-			type="String" format="GrepFindSyntax" default="error|warning|sorry"
-			readonly="false" visible="true" />
-
 		<parameter id="GrepSkip1" label="Grep skip pattern" type="String"
 			format="GrepSkipSyntax" default="(null)" readonly="false" visible="true" />
 			
@@ -309,6 +303,24 @@
 			label="Verilog parameters definition to be included in the test fixture"
 			readonly="false" visible="true" />
 			
+        <parameter id="parsers_path" label="Parsers Path" tooltip= "parsers directory in plugins"
+        			default="%%ParsersPath" visible="true" omit="" type="String" format="CopyValue"/>
+        <parameter id="parser_name" label="Parser name" tooltip= "Cocotb/Python output parser path"
+        			default="parser_cocotb_icarus.py" visible="true" omit="" type="String" format="CopyValue"/>
+        			
+
+        <parameter	id="PatternErrors"   label="Errors" tooltip= "Regular expression for error messages"
+        			default="(.*):([0-9]+): [a-z_\- ]*error[: ]?(.*)"
+        			visible="true"  type="String" format="CopyValue"/>
+        <parameter	id="PatternWarnings"   label="Warnings" tooltip= "Regular expression for warnings messages"
+        			default="(.*):([0-9]+): [a-z_\- ]*[warning|sorry]: (.*)"
+        			visible="true"  type="String" format="CopyValue"/>
+        <parameter	id="PatternInfo"   label="Info" tooltip= "Regular expression for info messages"
+        			default=".*[\s.](\w*\.py):([0-9]+)\s*\S*\s*\S*\s*(.*)"
+        			visible="true"  type="String" format="CopyValue"/>
+
+			
+			
 <!-- Temporary inserted into the command line, will be removed -->
 
 		<parameter id="Param_1" label="Param_1" type="String"
@@ -367,6 +379,38 @@
 			
 		<parameter id="CocotbRoot" outid="COCOTB" type="String"
 			 format="NameEqValue" default="%%PluginRoot/%CocotbRootRel" visible="false" />
+		<if ShowWarnings="true">
+		     <parameter id="GrepWarning" type="String" label = "GrepWarning"
+			 format="CopyValue" default="|warning|WARNING" visible="true" />
+		</if>	 
+		<if-not ShowWarnings="true">
+		     <parameter id="GrepWarning" type="String" label = "GrepWarning"
+			 format="CopyValue" default="" visible="true" />
+		</if-not>
+			 
+		<if ShowInfo="true">
+		     <parameter id="GrepInfo" type="String" label = "GrepInfo"
+			 format="CopyValue" default="|info|INFO" visible="true" />
+		</if>	 
+		<if-not ShowInfo="true">
+		     <parameter id="GrepInfo" type="String" label = "GrepInfo"
+			 format="CopyValue" default="" visible="true" />
+		</if-not>	 
+
+		<if COCOTB_DEBUG="true">
+		     <parameter id="GrepDebug" type="String" label = "GrepDebug"
+			 format="CopyValue" default="|DEBUG" visible="true" />
+		</if>	 
+		<if-not COCOTB_DEBUG="true">
+		     <parameter id="GrepDebug" type="String" label = "GrepDebug"
+			 format="CopyValue" default="" visible="true" />
+		</if-not>	 
+
+
+        <parameter id="GrepEWI" type="String" label = "GrepWarning"
+		format="GrepFindSyntax" default="error|ERROR%GrepWarning%GrepInfo%GrepDebug" visible="true" readonly="true"/>
+
+
 
 		<input>
 			<group name="cocotb" label="Cocotb">
@@ -401,13 +445,26 @@
 				<!--"ShowWaves"   -->
 				"ShowNoProblem"
 				"ShowWarnings"
+				"ShowInfo"
 				"RemoveBugs"
 				"SaveLogsPreprocessor"
 				"SaveLogsSimulator"
 				"v"
 				"legacy_model"
 				"no_specify"
+				"---"
+				"FilterParser"
+				"parsers_path"
+				"parser_name"
+<!-- 			"GrepWarning"
+				"GrepInfo"  -->
+				"GrepEWI"
+		        "PatternErrors"
+		        "PatternWarnings"
+		        "PatternInfo"
+				
 			</group>
+			
 			<group name="Advanced" label="Advanced">
 				"Param_PreExe"
 				"Param_Exe"
@@ -419,8 +476,6 @@
 				"LogFile"
 				"OutFile"
 				"FstDumpFile"
-				"GrepFindErr"
-				"GrepFindErrWarn"
 				"GrepSkip1"
 				"Param_1"
 				"Param_2"
@@ -447,11 +502,12 @@
 	- 			"else echo 'FAILURE';"
 				"fi"
 		    </line>
-		<!-- TODO: watch for new lines inserted inside quoted tokens during autoformat - they break output
-		Maybe add filter to the code to transform white spaces -->
+
 			<line name="command_line"
 			 sep=" "
-			 success = "SIMULATION FINISHED SUCCESSFULLY"
+			 errors=   "PatternErrors"
+			 warnings= "PatternWarnings"
+			 info=     "PatternInfo"
 			 failure = "ERROR|CRITICAL"
 	         >
 				"%Param_Shell_Options"
@@ -463,7 +519,25 @@
 			    "cd %CocotbFilesDir"
 			    ";"
 				<if IVerilogActionIndex="0">
-					"make sim;"
+					"make sim"
+					<if ShowNoProblem="false">
+						"2&gt;&amp;1"
+					</if>
+					<if SaveLogsPreprocessor="true">
+						"| tee -a"
+						"%%ProjectPath/%LogFileFull"
+					</if>
+					<if FilterParser="true">
+					    "|python -u %parsers_path/%parser_name"
+					</if>					
+ 				    <if ShowNoProblem="false">
+ 				        "%GrepEWI"
+						<if RemoveBugs="true">
+							"%GrepSkip1"
+						</if>
+					</if>
+					
+					";"
 				    "if [ $? -ne 0 ]; then  echo 'SIMULATION FAILED'; sleep 1; exit 1; fi;"
 					<!-- Add gtkwave here -->
 					<if ShowWaves="true">
@@ -482,7 +556,7 @@
 				<if IVerilogActionIndex="2">
 					"make clean;"
 				</if>
-			    "if [ $? -eq 0 ]; then  echo 'SIMULATION FINISHED SUCCESSFULLY'; else echo 'SIMULATION FAILED'; fi;"
+			    "if [ $? -eq 0 ]; then  echo 'SIMULATION FINISHED'; else echo 'SIMULATION FAILED'; fi;"
 			    "sleep 1"
 			    
 			</line>
@@ -515,10 +589,6 @@
 				"%CocotbCOMPILE_ARGS"
 				<!-- Always include top project dir (where IVERILOG_INCLUDE.v is) -->
 				"COMPILE_ARGS += -DCOCOTB"
-				<!-- So code may use PROJECT_ROOT_PATH to reference data files
-				     Does not work when passing string parameters, had to use include file -->
-<!-- 			"COMPILE_ARGS += -DROOTPATH=\&quot;%%ProjectPath\&quot;"
-				"COMPILE_ARGS += -DFSTNAME=\&quot;%%ParamValue-%%BuildStamp.fst\&quot;"-->
 				
 				<if ShowWaves="true">
 					"COMPILE_ARGS += -DTRACE"