Debug output for lexical analyzer

In case of error messages like:
    input buffer overflow, can't enlarge buffer because scanner uses REJECT
it is not always directly clear from which lexical analyzer (.l file) this problem comes.
This patch helps to find these problems and does the following things:
- when using the option -d lex with doxygen each time a lexical analyzer is called at the start a line like the following line will be given:
    Entering lexical analyzer: pre.l (for: ..../file.c)
  and at the end:
    Finished lexical analyzer: pre.l (for: ..../file.c)
- in case the lexical analyzer has been translated with the -d option of lex / flex the above mentioned lines will be given as part of the lexical analyzer output (to stderr) and look like:
    --entering lexical analyzer: pre.l (for: ..../file.c)
    --finished lexical analyzer: pre.l (for: ..../file.c)

src/code.l

@@ -3546,7 +3546,11 @@ void parseCCode(CodeOutputInterface &od,const char *className,const QCString &s,
 {
   //printf("***parseCode() exBlock=%d exName=%s fd=%p className=%s searchCtx=%s\n",
   //      exBlock,exName,fd,className,searchCtx?searchCtx->name().data():"<none>");
+
   if (s.isEmpty()) return;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
+
   TooltipManager::instance()->clearTooltips();
   if (g_codeClassSDict==0)
   {
@@ -3637,6 +3641,8 @@ void parseCCode(CodeOutputInterface &od,const char *className,const QCString &s,
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 

src/commentcnv.l

@@ -952,6 +952,7 @@ void convertCppComments(BufStr *inBuf,BufStr *outBuf,const char *fileName)
   g_condStack.clear();
   g_condStack.setAutoDelete(TRUE);
 
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   isFixedForm = FALSE;
   if (g_lang==SrcLangExt_Fortran)
   {
@@ -981,6 +982,7 @@ void convertCppComments(BufStr *inBuf,BufStr *outBuf,const char *fileName)
     g_outBuf->at(g_outBuf->curPos())='\0';
     msg("-------------\n%s\n-------------\n",g_outBuf->data());
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 

src/commentscan.l

@@ -2818,6 +2818,7 @@ bool parseCommentBlock(/* in */     ParserInterface *parser,
   g_spaceBeforeCmd.resize(0);
   g_spaceBeforeIf.resize(0);
 
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName ? fileName.data(): NULL);
   if (!current->inbodyDocs.isEmpty() && isInbody) // separate in body fragments
   {
     current->inbodyDocs+="\n\n";
@@ -2894,6 +2895,7 @@ bool parseCommentBlock(/* in */     ParserInterface *parser,
   //printf("position=%d parseMore=%d newEntryNeeded=%d\n",
   //  position,parseMore,newEntryNeeded);
 
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName ? fileName.data(): NULL);
   return parseMore;
 }
 

src/constexp.l

@@ -21,6 +21,7 @@
 #include "constexp.h"  
 #include "cppvalue.h"
 #include "ce_parse.h" // generated header file
+#include "message.h"
 
 #define YY_NEVER_INTERACTIVE 1
 #define YY_NO_INPUT 1
@@ -105,6 +106,7 @@ CONSTSUFFIX ([uU][lL]?[lL]?)|([lL][lL]?[uU]?)
 
 bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   //printf("Expression: `%s'\n",s.data());
   g_constExpFileName = fileName;
   g_constExpLineNr = lineNr;
@@ -113,6 +115,7 @@ bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
   constexpYYrestart( constexpYYin );
   constexpYYparse();
   //printf("Result: %ld\n",(long)g_resultValue);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
   return (long)g_resultValue!=0;
 }
 

src/debug.cpp

@@ -46,6 +46,7 @@ static LabelMap s_labels[] =
   { "extcmd",       Debug::ExtCmd       },
   { "markdown",     Debug::Markdown     },
   { "filteroutput", Debug::FilterOutput },
+  { "lex",          Debug::Lex },
   { 0,             (Debug::DebugMask)0  }
 };
 

src/debug.h

@@ -36,7 +36,8 @@ class Debug
                      Time         = 0x00000200,
                      ExtCmd       = 0x00000400,
                      Markdown     = 0x00000800,
-                     FilterOutput = 0x00001000
+                     FilterOutput = 0x00001000,
+                     Lex          = 0x00002000
                    };
     static void print(DebugMask mask,int prio,const char *fmt,...);
     static int  setFlag(const char *label);

src/declinfo.l

@@ -231,6 +231,7 @@ ID	"$"?([a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*)|(@[0-9]+)
 void parseFuncDecl(const QCString &decl,bool objC,QCString &cl,QCString &t,
                    QCString &n,QCString &a,QCString &ftl,QCString &exc)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, NULL);
   inputString   = decl;
   //printf("Input=`%s'\n",inputString);
   if (inputString==0) return;
@@ -307,6 +308,7 @@ void parseFuncDecl(const QCString &decl,bool objC,QCString &cl,QCString &t,
   //printf("type=`%s' class=`%s' name=`%s' args=`%s'\n",
   //        t.data(),cl.data(),n.data(),a.data());
 
+  printlex(yy_flex_debug, FALSE, __FILE__, NULL);
   return;
   
   

src/defargs.l

@@ -55,6 +55,7 @@
 #include "entry.h"
 #include "util.h"
 #include "arguments.h"
+#include "message.h"
   
 #define YY_NEVER_INTERACTIVE 1
 #define YY_NO_INPUT 1
@@ -527,6 +528,7 @@ void stringToArgumentList(const char *argsString,ArgumentList* al,QCString *extr
 {
   if (al==0) return; 
   if (argsString==0) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, NULL);
 
   g_copyArgValue=0;
   g_curArgDocs.resize(0);
@@ -549,6 +551,7 @@ void stringToArgumentList(const char *argsString,ArgumentList* al,QCString *extr
   defargsYYlex();
   if (extraTypeChars) *extraTypeChars=g_extraTypeChars;
   //printf("stringToArgumentList(%s) result=%s\n",argsString,argListToString(al).data());
+  printlex(yy_flex_debug, FALSE, __FILE__, NULL);
 }
 
 #if !defined(YY_FLEX_SUBMINOR_VERSION) 

src/doctokenizer.l

@@ -1189,6 +1189,7 @@ void doctokenizerYYFindSections(const char *input,Definition *d,
                                 MemberGroup *mg,const char *fileName)
 {
   if (input==0) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   g_inputString = input;
   //printf("parsing --->`%s'<---\n",input);
   g_inputPos    = 0;
@@ -1198,6 +1199,7 @@ void doctokenizerYYFindSections(const char *input,Definition *d,
   BEGIN(St_Sections);
   doctokenizerYYlineno = 1;
   doctokenizerYYlex();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void doctokenizerYYinit(const char *input,const char *fileName)

src/fortrancode.l

@@ -1115,6 +1115,7 @@ void parseFortranCode(CodeOutputInterface &od,const char *className,const QCStri
   (void)className;
 
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   g_code = &od;
   g_inputString   = s;
@@ -1174,6 +1175,7 @@ void parseFortranCode(CodeOutputInterface &od,const char *className,const QCStri
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 

src/fortranscanner.l

@@ -2339,7 +2339,12 @@ void FortranLanguageScanner::parseInput(const char *fileName,
                                         QStrList & /*filesInSameTranslationUnit*/)
 {
   g_thisParser = this;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
+
   ::parseMain(fileName,fileBuf,root);
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void FortranLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,

src/message.cpp

@@ -211,3 +211,31 @@ void err(const char *fmt, ...)
   vfprintf(warnFile, (QCString(error_str) + fmt).data(), args);
   va_end(args); 
 }
+
+void printlex(int dbg, bool enter, const char *lexName, const char *fileName)
+{
+  char *enter_txt = "entering";
+  char *enter_txt_uc = "Entering";
+
+  if (!enter)
+  {
+    enter_txt = "finished";
+    enter_txt_uc = "Finished";
+  }
+
+  if (dbg)
+  {
+    if (fileName)
+      fprintf(stderr,"--%s lexical analyzer: %s (for: %s)\n",enter_txt, lexName, fileName);
+    else
+      fprintf(stderr,"--%s lexical analyzer: %s\n",enter_txt, lexName);
+  }
+  else
+  {
+    if (fileName)
+      Debug::print(Debug::Lex,0,"%s lexical analyzer: %s (for: %s)\n",enter_txt_uc, lexName, fileName);
+    else
+      Debug::print(Debug::Lex,0,"%s lexical analyzer: %s\n",enter_txt_uc, lexName);
+  }
+
+}

src/message.h

@@ -29,4 +29,5 @@ extern void warn_uncond(const char *fmt, ...);
 extern void err(const char *fmt, ...);
 void initWarningFormat();
 
+extern void printlex(int dbg, bool enter, const char *lexName, const char *fileName);
 #endif

src/pre.l

@@ -2969,6 +2969,7 @@ void cleanUpPreprocessor()
 
 void preprocessFile(const char *fileName,BufStr &input,BufStr &output)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   uint orgOffset=output.curPos();
   //printf("##########################\n%s\n####################\n",
   //    input.data());
@@ -3177,6 +3178,7 @@ void preprocessFile(const char *fileName,BufStr &input,BufStr &output)
     }
   }
   DefineManager::instance().endContext();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void preFreeScanner()

src/pycode.l

@@ -1434,6 +1434,7 @@ void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
   
   //--------------------------------------
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   g_code = &od;
   g_inputString   = s;
@@ -1494,6 +1495,7 @@ void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 

src/pyscanner.l

@@ -1686,7 +1686,9 @@ void PythonLanguageScanner::parseInput(const char *fileName,
                                        QStrList & /*filesInSameTranslationUnit*/)
 {
   g_thisParser = this;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   ::parseMain(fileName,fileBuf,root);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 
   // May print the AST for debugging purposes
   // printAST(global_root);

src/scanner.l

@@ -6893,8 +6893,13 @@ void CLanguageScanner::parseInput(const char *fileName,
                                   QStrList & filesInSameTranslationUnit)
 {
   g_thisParser = this;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
+
   ::parseMain(fileName,fileBuf,root,
               sameTranslationUnit,filesInSameTranslationUnit);
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void CLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,

src/tclscanner.l

@@ -127,7 +127,7 @@ int TclFindElement(
     goto done;
   }
 
-  if (*p == '{') 
+  if (*p == '{')   /* } to keep vi happy */
   {
     openBraces = 1;
     p++;
@@ -2541,6 +2541,7 @@ tcl_inf("%s\n",fileName);
 
   tcl.input_string = input;
   if (tcl.input_string.length()<1) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
 
   msg("Parsing %s...\n",fileName);
   groupEnterFile(fileName,yylineno);
@@ -2554,6 +2555,7 @@ tcl_inf("%s\n",fileName);
   groupLeaveFile(tcl.file_name,yylineno);
   root->program.resize(0);
   myFile.close();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 //! Parse file and codify.
@@ -2583,6 +2585,7 @@ void TclLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,
   (void)collectXRefs;
 
   if (input.length()<1) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileDef ? fileDef->fileName().data(): NULL);
   tcl.input_string = input;
 
   QCString myNs="";
@@ -2655,6 +2658,7 @@ tcl_inf("%s (%d,%d) %d %d\n",myStr.ascii(),startLine,endLine,isExampleBlock,inli
   tcl.cl.clear();
   tcl.fn.clear();
   tcl.entry.clear();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileDef ? fileDef->fileName().data(): NULL);
 }
 
 bool TclLanguageScanner::needsPreprocessing(const QCString &extension)

src/vhdlcode.l

@@ -1517,6 +1517,7 @@ void parseVhdlCode(CodeOutputInterface &od,const char *className,const QCString
 {
   //printf("***parseCode() exBlock=%d exName=%s fd=%p\n",exBlock,exName,fd);
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   if (memberDef)
   {
@@ -1599,6 +1600,7 @@ void parseVhdlCode(CodeOutputInterface &od,const char *className,const QCString
     g_sourceFileDef=0;
   }
   g_startCode=FALSE;
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
 }
 
 void codeFreeVhdlScanner()

src/vhdlscanner.l

@@ -763,6 +763,7 @@ void VHDLLanguageScanner::parseInput(const char *fileName,
     VhdlDocGen::parseUCF(fileBuf,root,yyFileName,TRUE); 
     return; 
   }
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
 
   ::parserInit();
   yycont=getVhdlCont();
@@ -791,6 +792,7 @@ void VHDLLanguageScanner::parseInput(const char *fileName,
   free(lineIndex);
   inputFile.close();
   mapLibPackage(root);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void VHDLLanguageScanner::parseCode(CodeOutputInterface &codeOutIntf,