Changed timestamp in the filenames format to be more human-friendly,

fixed some bugs and mitigated the other (closeInputStream() that does
not actually flush completely) 

src/com/elphel/vdt/core/launching/RunningBuilds.java

@@ -20,17 +20,49 @@ package com.elphel.vdt.core.launching;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+
+import org.eclipse.debug.core.IStreamListener;
+import org.eclipse.debug.core.model.IStreamMonitor;
 import org.eclipse.ui.console.IConsole;
 import org.eclipse.ui.console.ConsolePlugin;
 import org.eclipse.ui.console.IConsoleManager;
+
 import com.elphel.vdt.veditor.VerilogPlugin;
 import com.elphel.vdt.veditor.preference.PreferenceStrings;
+
 import org.eclipse.ui.console.IConsoleListener;
 
 public class RunningBuilds {
+	public class MonListener{
+		private IStreamMonitor monitor;
+		private IStreamListener listener;
+		public IStreamMonitor getMonitor() {
+			return monitor;
+		}
+		public IStreamListener getListener() {
+			return listener;
+		}
+		public MonListener(IStreamMonitor monitor, IStreamListener listener) {
+			super();
+			this.monitor = monitor;
+			this.listener = listener;
+		}
+		public void finalize() throws Throwable{
+			if ((monitor!=null) && (listener!=null)){
+				monitor.removeListener(listener);
+			}
+			monitor=null;
+			listener=null;
+			super.finalize();
+		}
+	}
+    private Map<IConsole,MonListener> parserListeners=null; // consoles mapped to pairs of monitors and listeners
+                                                            // that should be disconnected when parser is terminated
+
 //	int nextBuildStep=0;
 	private final Map<String, VDTRunnerConfiguration> unfinishedBuilds;
 	public RunningBuilds(){
+		parserListeners= new ConcurrentHashMap<IConsole,MonListener>();
 		unfinishedBuilds = new ConcurrentHashMap<String, VDTRunnerConfiguration>();
 		IConsoleManager manager = ConsolePlugin.getDefault().getConsoleManager();
 // This is not used, just for testing
@@ -52,10 +84,30 @@ public class RunningBuilds {
 						System.out.println("--- Removed: "+consoles[i].getName());
 					}
 					//					unfinishedBuilds.remove(consoles[i]);
+					removeMonListener(consoles[i]); // remove listeners that provided input data for parsers
 					removeConsole(consoles[i]);
 				}
 			}
 		});
+		
+	}
+	
+	public void addMonListener(IConsole parserConsole, IStreamMonitor monitor, IStreamListener listener){
+		parserListeners.put(parserConsole, new MonListener(monitor, listener));
+	}
+	private void removeMonListener(IConsole parserConsole){
+		MonListener monListener=parserListeners.remove(parserConsole);
+		if (monListener!=null){
+			if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_LAUNCHING)) {
+				System.out.println("--- Removing listener from the terminated parser console "+parserConsole.getName());
+			}
+			try {
+				monListener.finalize();
+			} catch (Throwable e) {
+				System.out.println("Failed to finalize monListener for console "+parserConsole.getName());
+				e.printStackTrace();
+			}
+		}
 	}
 	
 	public String findConsoleParent(IConsole console){

src/com/elphel/vdt/core/launching/ToolLogFile.java

@@ -55,8 +55,11 @@ public class ToolLogFile {
 	private IFile targetErrIFile;
 	private boolean hasOut;
 	private boolean hasErr;
-
 	private boolean singleFile;
+	private int errBytes;
+	private int outBytes;
+	
+	
 	private boolean debugPrint;
 	public static IFolder getDir(String logDir){
 		IProject project = SelectedResourceManager.getDefault().getSelectedProject(); // should not be null when we got here
@@ -161,6 +164,8 @@ public class ToolLogFile {
 		targetErrIFile = singleFile? targetOutIFile : iLogFolder.getFile(baseNameErr+buildStampWithSep+ext);
 		
 		if (writeMode) {
+			outBytes=0; // jsut for debugging
+			errBytes=0;
 			byte [] emptyBA={};
 
 			IFile linkOutIFile=   iLogFolder.getFile(baseNameOut+ext);
@@ -249,6 +254,7 @@ public class ToolLogFile {
 		if (!hasOut ||(logOutWriter==null)) return; // do nothing
 		try {
 			logOutWriter.append(string);
+			errBytes+=string.length();
 //			if(debugPrint) System.out.println("out->out: "+string);
 
 		} catch (IOException e) {
@@ -263,6 +269,7 @@ public class ToolLogFile {
 		try {
 			if (singleFile) {
 				logOutWriter.append(string);
+				outBytes+=string.length();
 //				if(debugPrint) System.out.println("err->out: "+string);
 			} else {
 				logErrWriter.append(string);
@@ -278,20 +285,22 @@ public class ToolLogFile {
 	}
 
 	public void closeOut(){
-		if (logOutWriter!=null)
+		if (logOutWriter!=null) {
 			try {
 				logOutWriter.close();
-				if(debugPrint) System.out.println("closeOut()");
+				if(debugPrint) System.out.println("closeOut(), wrote "+outBytes+" bytes");
 			} catch (IOException e) {
 				System.out.println("Failed to close log file "+targetOutIFile.toString());
 			}
+
+		}
 	}
 
 	public void closeErr(){
 		if (logErrWriter!=null)
 			try {
 				logErrWriter.close();
-				if(debugPrint) System.out.println("closeErr()");
+				if(debugPrint) System.out.println("closeErr(), wrote "+errBytes+" bytes");
 			} catch (IOException e) {
 				System.out.println("Failed to close error log file "+targetErrIFile.toString());
 			}

src/com/elphel/vdt/core/launching/VDTConsolePlayback.java

@@ -27,7 +27,6 @@ import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.debug.core.ILaunch;
 import org.eclipse.debug.core.model.IProcess;
 import org.eclipse.debug.core.model.IStreamsProxy2;
-import org.eclipse.swt.widgets.Display;
 
 import com.elphel.vdt.core.tools.contexts.BuildParamsItem;
 import com.elphel.vdt.veditor.VerilogPlugin;
@@ -179,10 +178,12 @@ public class VDTConsolePlayback{
         // Read logs should be fast, not sure how much will be buffered in IStreamsProxy2, so sending logs
         // one after another in the same thread
         if (errBufReader!=null){
+        	int dbgBytes=0;
         	String line;
         	try {
         		while ((line = errBufReader.readLine()) != null){
         			fSendErrorsToStreamProxy.write(line+"\n");
+        			dbgBytes+=line.length()+1;
         			//if(debugPrint) System.out.println ("err->>"+line);
 
         		}
@@ -191,44 +192,57 @@ public class VDTConsolePlayback{
         	} finally {
         		try {
 					errBufReader.close();
-//					if(debugPrint) System.out.println ("err->>CLOSED");
+					if(debugPrint) System.out.println ("err->>CLOSED, got "+dbgBytes+" bytes");
 				} catch (IOException e) {
 	        		System.out.println ("Failed to close "+errLogName+" BufferedReader");
 				}
         	}
         }
         
-        
         if (outBufReader!=null){
+        	int dbgBytes=0;
         	String line;
         	try {
         		while ((line = outBufReader.readLine()) != null){
         			fSendOutputToStreamProxy.write(line+"\n");
-//        			if(debugPrint) System.out.println ("out->>"+line);
+        			dbgBytes+=line.length()+1;
+ //       			if(debugPrint) System.out.println ("out->>"+line);
         		}
         	} catch (IOException e) {
         		System.out.println ("Failed to replay "+outLogName+" to output parser");
         	} finally {
         		try {
 					outBufReader.close();
-//					if(debugPrint) System.out.println ("out->>CLOSED");
+					if(debugPrint) System.out.println ("out->>CLOSED, got "+dbgBytes+" bytes"); // bytes OK, always the same
 				} catch (IOException e) {
 	        		System.out.println ("Failed to close "+outLogName+" BufferedReader");
 				}
         	}
         }
 
-        try {
-        	if (stdoutStreamProxy!=null) stdoutStreamProxy.closeInputStream();
+        if (stdoutStreamProxy!=null) try {
+        	// Checked that all is sent through write() method, by end of data is often lost when calling closeInputStream() too soon AFTER
+        	// Less visible when the source is doing something, more - here, when it is just a play back
+        	if(debugPrint) System.out.println("mitigating possible closeInputStream() bug - sleeping "+VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY+" ms");
+        	Thread.sleep(VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY);
+        	stdoutStreamProxy.closeInputStream();
+//        	System.out.println ("closed output stream proxy");
         } catch (IOException e){
         	System.out.println ("Failed to close output stream proxy");
+        } catch (InterruptedException e) {
 		}
         // next uses stderrStreamProxy, not fSendErrorsToStreamProxy as it can be the same as fSendOutputToStreamProxy
-        try {
-        	if (stderrStreamProxy!=null) stderrStreamProxy.closeInputStream(); 
+        if (stderrStreamProxy!=null) try {
+        	if(debugPrint) System.out.println("mitigating possible closeInputStream() bug - sleeping "+VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY+" ms");
+        	Thread.sleep(VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY);
+        	stderrStreamProxy.closeInputStream(); 
+        	System.out.println ("closed error stream proxy"); // ???
         } catch (IOException e){
         	System.out.println ("Failed to close error stream proxy");
+        } catch (InterruptedException e) {
+			e.printStackTrace();
 		}
+
         return true;
 	}
     

src/com/elphel/vdt/core/launching/VDTConsoleRunner.java

@@ -23,13 +23,13 @@ import java.util.Timer;
 import java.util.TimerTask;
 
 import org.eclipse.debug.internal.ui.views.console.ProcessConsole;
+import org.eclipse.debug.ui.DebugUITools;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.debug.core.DebugException;
 import org.eclipse.debug.core.ILaunch;
 import org.eclipse.debug.core.IStreamListener;
 import org.eclipse.debug.core.model.IProcess;
-
 import org.eclipse.debug.core.model.IStreamMonitor;
 import org.eclipse.debug.core.model.IStreamsProxy2;
 import org.eclipse.swt.graphics.Color;
@@ -52,8 +52,9 @@ public class VDTConsoleRunner{
 	private IProcess processErr=null; //*
 	private IProcess processOut=null; //*
 	private IStreamsProxy2 sendErrorsToStreamProxy=null;
-	private Object errorListener=null; //+
-	private Object outputListener=null; //+
+//	private Object errorListener=null; //+
+	private IStreamListener errorListener=null; //+
+	private IStreamListener outputListener=null; //+
     private IStreamsProxy2 consoleInStreamProxy= null; //+
     private Timer timer;
     private IStreamsProxy2 stdoutStreamProxy=null;
@@ -236,12 +237,15 @@ public class VDTConsoleRunner{
         		if (runConfig.addConsoleText(text)){
         			if (debugPrint)  System.out.println("Got finish sequence");
         			// TODO: launch continuation of the build process
-        			finishConsolescript();
+        			finishConsolescript(); // got here when computer running Vivado was disconnected
         		}
         	}
         };
-        consoleErrStreamMonitor.addListener((IStreamListener) errorListener);       		
-        //      	}
+        VDTLaunchUtil.getRunner().getRunningBuilds().addMonListener( // to remove listener when parser is terminated
+        		DebugUITools.getConsole(processErr), //IConsole parserConsole,
+        		consoleErrStreamMonitor,
+        		errorListener);
+        consoleErrStreamMonitor.addListener(errorListener);
         outputListener=null;
         //        if (fSendOutputToStreamProxy!=null){
         consoleOutStreamMonitor=consoleInStreamProxy.getOutputStreamMonitor();
@@ -264,7 +268,11 @@ public class VDTConsoleRunner{
         		}
         	}
         };
-        consoleOutStreamMonitor.addListener((IStreamListener) outputListener );
+        VDTLaunchUtil.getRunner().getRunningBuilds().addMonListener( // to remove listener when parser is terminated
+        		DebugUITools.getConsole(processOut), //IConsole parserConsole,
+        		consoleOutStreamMonitor,
+        		outputListener);
+        consoleOutStreamMonitor.addListener(outputListener );
         //       }
         //Problems occurred when invoking code from plug-in: "org.eclipse.ui.console".
         //Exception occurred during console property change notification.
@@ -303,6 +311,15 @@ public class VDTConsoleRunner{
     public void finishConsolescript() {
         final boolean debugPrint=VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_LAUNCHING);
     	if (debugPrint) System.out.println("finishConsolescript()");
+		String playBackStamp=runConfig.getPlayBackStamp();
+		if (playBackStamp!=null){
+			// happened when Vivaod console was disconnected with old console listener still attached
+			// they should be removed when a parser process is terminated
+			
+			
+			System.out.println("Wrong, it should be playback, not run, as playBackStamp = "+playBackStamp+ "(not null)");
+			return;
+		}
 		if (timer!=null){
 			timer.cancel();
 		}
@@ -316,11 +333,11 @@ public class VDTConsoleRunner{
     	}
     	if (errorListener !=null) { // disconnect error stream listener
     		IStreamMonitor consoleErrorStreamMonitor=consoleInStreamProxy.getOutputStreamMonitor();
-    		consoleErrorStreamMonitor.removeListener((IStreamListener) errorListener);
+    		consoleErrorStreamMonitor.removeListener(errorListener);
     	}
     	if (outputListener !=null) { // disconnect output stream listener
     		IStreamMonitor consoleOutStreamMonitor=consoleInStreamProxy.getOutputStreamMonitor();
-    		consoleOutStreamMonitor.removeListener((IStreamListener) outputListener);
+    		consoleOutStreamMonitor.removeListener(outputListener);
     	}
     	// terminate parser(s). Do those console listeners (parsers) have to be removed too?
     	// TODO: Maybe wait for the process (small time) to terminate by disconnecting it's stdin
@@ -330,6 +347,12 @@ public class VDTConsoleRunner{
     	
     	if (stderrStreamProxy!=null){
     		try {
+            	// Checked that all is sent through write() method, by end of data is often lost when calling closeInputStream() too soon AFTER
+            	if(debugPrint) System.out.println("mitigating possible closeInputStream() bug - sleeping "+VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY+" ms");
+            	try {
+					Thread.sleep(VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY);
+				} catch (InterruptedException e) {
+				}
     			stderrStreamProxy.closeInputStream();
 			} catch (IOException e) {
 				System.out.println("Failed to disconnect stdin of the processErr parser process");
@@ -344,6 +367,12 @@ public class VDTConsoleRunner{
     	}
     	if (stdoutStreamProxy!=null){
     		try {
+            	// Checked that all is sent through write() method, by end of data is often lost when calling closeInputStream() too soon AFTER
+            	if(debugPrint) System.out.println("mitigating possible closeInputStream() bug - sleeping "+VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY+" ms");
+            	try {
+					Thread.sleep(VDTLaunchUtil.CLOSE_INPUT_STREAM_DELAY);
+				} catch (InterruptedException e) {
+				}
     			stdoutStreamProxy.closeInputStream();
 			} catch (IOException e) {
 				System.out.println("Failed to disconnect stdin of the processOut parser process");

src/com/elphel/vdt/core/launching/VDTLaunchConfigurationDelegate.java

@@ -120,6 +120,7 @@ public class VDTLaunchConfigurationDelegate implements ILaunchConfigurationDeleg
         if (playBackStamp==null){
         	runner.resumeLaunch(consoleName); // actual run of the tools
         } else {
+        	runConfig.setBuildStep(-1); // to cause errors if will try to continue
         	runner.logPlaybackLaunch(consoleName); // tool logs playback with parsing
         }
         return;

src/com/elphel/vdt/core/launching/VDTLaunchUtil.java

@@ -59,6 +59,7 @@ import com.elphel.vdt.ui.MessageUI;
  */
 
 public class VDTLaunchUtil {
+	public final static int CLOSE_INPUT_STREAM_DELAY = 1000; // ms
     private static VDTRunner toolRunner;
 
     /**

src/com/elphel/vdt/core/launching/VDTRunner.java

@@ -306,7 +306,10 @@ public class VDTRunner {
 		
 		BuildParamsItem[] argumentsItemsArray = runConfig.getArgumentsItemsArray(); // uses already calculated
 		runConfig.canceTimers(); // If some timers were set, but a task finished earlier
-		int numItem=runConfig.getBuildStep();
+//		int numItem=runConfig.getBuildStep();
+		// made buildStep for logs negative (it does not need to sleep and does everything in one call)
+		// to catch stray resumes
+		int numItem=0; //runConfig.getBuildStep();
 		if (debugPrint) System.out.println("--------- re-playing log from "+ consoleName+", numItem="+numItem+" ------------");
 		ILaunch launch=runConfig.getLaunch();
 		IProgressMonitor monitor=runConfig.getMonitor();

src/com/elphel/vdt/core/launching/VDTRunnerConfiguration.java

@@ -18,13 +18,17 @@
 package com.elphel.vdt.core.launching;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.debug.core.ILaunch;
 import org.eclipse.debug.core.ILaunchConfiguration;
+import org.eclipse.debug.core.IStreamListener;
+import org.eclipse.debug.core.model.IStreamMonitor;
 //import org.eclipse.debug.ui.console.IConsole;
 import org.eclipse.ui.console.IConsole;
 
@@ -45,6 +49,29 @@ import com.elphel.vdt.ui.MessageUI;
  */
 
 public class VDTRunnerConfiguration {
+	public class MonListener{
+		private IStreamMonitor monitor;
+		private IStreamListener listener;
+		public IStreamMonitor getMonitor() {
+			return monitor;
+		}
+		public IStreamListener getListener() {
+			return listener;
+		}
+		public MonListener(IStreamMonitor monitor, IStreamListener listener) {
+			super();
+			this.monitor = monitor;
+			this.listener = listener;
+		}
+		public void finalize() throws Throwable{
+			if ((monitor!=null) && (listener!=null)){
+				monitor.removeListener(listener);
+			}
+			monitor=null;
+			listener=null;
+			super.finalize();
+		}
+	}
 
 	private String   toolToLaunch;
 	private String   toolProjectPath;
@@ -83,10 +110,13 @@ public class VDTRunnerConfiguration {
     
     private String playBackStamp=null; // timestamp of the logs to play back ("" for latest), or
                                        // null for normal running tools
+    private Set<IConsole> consoles=null;  // parser consoles opened for this console
+    
+    private Map<IConsole,MonListener> parserListeners=null; // consoles mapped to pairs of monitors and listeners
+                                                            // that should be disconnected when parser is terminated
     
     
     
-    private Set<IConsole> consoles=null;  // parser consoles opened for this console
 	private VDTConsoleRunner consoleRunner=null;
 	private VDTProgramRunner programRunner=null;
 	private VDTConsolePlayback consolePlayback=null;
@@ -127,12 +157,28 @@ public class VDTRunnerConfiguration {
 		
 		this.consoleBuffer="";
 		this.consoles= new HashSet<IConsole>();
+		this.parserListeners= new ConcurrentHashMap<IConsole,MonListener>();
 		this.consoleRunner= new VDTConsoleRunner(this); // arguments here?
 		this.programRunner=new VDTProgramRunner(); // arguments here?
 		this.consolePlayback=new VDTConsolePlayback(this);
-
+	}
+	public void addMonListener(IConsole parserConsole, IStreamMonitor monitor, IStreamListener listener){
+		parserListeners.put(parserConsole, new MonListener(monitor, listener));
+	}
+	public void removeMonListener(IConsole parserConsole){
+		MonListener monListener=parserListeners.remove(parserConsole);
+		if (monListener!=null){
+			try {
+				monListener.finalize();
+			} catch (Throwable e) {
+				System.out.println("Failed to finalize monListener for console "+parserConsole.getName());
+				e.printStackTrace();
+			}
+		}
 	}
 	
+	
+	
 	public void setPlayBackStamp(String str){
 		playBackStamp=str;
 	}
@@ -152,12 +198,16 @@ public class VDTRunnerConfiguration {
 		return this.programRunner;
 	}
 	
-	public void addConsole(IConsole console){
+	public void addConsole(IConsole console){ // not used
 		consoles.add(console);
 	}
-	public void removeConsole(IConsole console){
+	public void removeConsole(IConsole console){ // from VDTRunnerConfiguration
 		consoles.remove(console);
 	}
+	//TODO: add Map<IConsole,
+	
+	
+	
 	public boolean noConsoles(){
 		return consoles.isEmpty();
 	}

src/com/elphel/vdt/core/tools/params/types/ParamTypeString.java

@@ -161,9 +161,9 @@ public class ParamTypeString extends ParamType {
     public String canonicalizeValue(String value) {
 // Try to convert file/dir parameters to project-relative
     	if ((kind == KIND.FILE) || (kind == KIND.FILE)) {
-    		if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_OTHER)) System.out.print("Converting \""+value+"\"to ");
+//    		if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_OTHER)) System.out.print("Converting \""+value+"\"to ");
     		value=tryProjectRelativePath(value);
-    		if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_OTHER)) System.out.println("\""+value+"\"");
+//    		if (VerilogPlugin.getPreferenceBoolean(PreferenceStrings.DEBUG_OTHER)) System.out.println("\""+value+"\"");
     	}
     	
         switch(caseSensitive) {

src/com/elphel/vdt/ui/variables/SelectedResourceManager.java

@@ -17,6 +17,8 @@
  *******************************************************************************/
 package com.elphel.vdt.ui.variables;
 
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.Stack;
 
 import com.elphel.vdt.VerilogUtils;
@@ -68,7 +70,8 @@ public class SelectedResourceManager implements IWindowListener, ISelectionListe
     private String fChosenShort=null; // last segment of the chosen resource name
     private IResource fChosenVerilogFile = null; // to keep fSelectedVerilogFile
     private int fChosenAction=0; // Chosen variant of running the tool
-    private long timestamp=0;
+//    private long timestamp=0;
+    private String timestamp;
     private String ignoreFilter=null;
 //    private Tool selectedTool=null; // last selected tool
  //   
@@ -273,12 +276,15 @@ public class SelectedResourceManager implements IWindowListener, ISelectionListe
     }
     
     public String setBuildStamp(){
-    	timestamp=System.nanoTime();
+//    	timestamp=System.nanoTime();
+    	//String fileName = new SimpleDateFormat("yyyyMMddHHmmssSSS'.txt'").format(new Date()));
+    	timestamp= new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date());
     	return getBuildStamp();
     }
 
     public String getBuildStamp(){
-    	return ""+timestamp;
+    	
+    	return timestamp;
     }