Merge pull request #55 from albert-github/feature/debug_lex

Debug output for lexical analyzer

src/code.l

@@ -3546,7 +3546,11 @@ void parseCCode(CodeOutputInterface &od,const char *className,const QCString &s,
 {
   //printf("***parseCode() exBlock=%d exName=%s fd=%p className=%s searchCtx=%s\n",
   //      exBlock,exName,fd,className,searchCtx?searchCtx->name().data():"<none>");
+
   if (s.isEmpty()) return;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
+
   TooltipManager::instance()->clearTooltips();
   if (g_codeClassSDict==0)
   {
@@ -3637,6 +3641,8 @@ void parseCCode(CodeOutputInterface &od,const char *className,const QCString &s,
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 

src/commentcnv.l

@@ -952,6 +952,7 @@ void convertCppComments(BufStr *inBuf,BufStr *outBuf,const char *fileName)
   g_condStack.clear();
   g_condStack.setAutoDelete(TRUE);
 
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   isFixedForm = FALSE;
   if (g_lang==SrcLangExt_Fortran)
   {
@@ -981,6 +982,7 @@ void convertCppComments(BufStr *inBuf,BufStr *outBuf,const char *fileName)
     g_outBuf->at(g_outBuf->curPos())='\0';
     msg("-------------\n%s\n-------------\n",g_outBuf->data());
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 

src/commentscan.l

@@ -2818,6 +2818,7 @@ bool parseCommentBlock(/* in */     ParserInterface *parser,
   g_spaceBeforeCmd.resize(0);
   g_spaceBeforeIf.resize(0);
 
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName ? fileName.data(): NULL);
   if (!current->inbodyDocs.isEmpty() && isInbody) // separate in body fragments
   {
     current->inbodyDocs+="\n\n";
@@ -2894,6 +2895,7 @@ bool parseCommentBlock(/* in */     ParserInterface *parser,
   //printf("position=%d parseMore=%d newEntryNeeded=%d\n",
   //  position,parseMore,newEntryNeeded);
 
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName ? fileName.data(): NULL);
   return parseMore;
 }
 

src/constexp.l

@@ -21,6 +21,7 @@
 #include "constexp.h"  
 #include "cppvalue.h"
 #include "ce_parse.h" // generated header file
+#include "message.h"
 
 #define YY_NEVER_INTERACTIVE 1
 #define YY_NO_INPUT 1
@@ -105,6 +106,7 @@ CONSTSUFFIX ([uU][lL]?[lL]?)|([lL][lL]?[uU]?)
 
 bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   //printf("Expression: `%s'\n",s.data());
   g_constExpFileName = fileName;
   g_constExpLineNr = lineNr;
@@ -113,6 +115,7 @@ bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
   constexpYYrestart( constexpYYin );
   constexpYYparse();
   //printf("Result: %ld\n",(long)g_resultValue);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
   return (long)g_resultValue!=0;
 }
 

src/debug.cpp

@@ -46,6 +46,7 @@ static LabelMap s_labels[] =
   { "extcmd",       Debug::ExtCmd       },
   { "markdown",     Debug::Markdown     },
   { "filteroutput", Debug::FilterOutput },
+  { "lex",          Debug::Lex },
   { 0,             (Debug::DebugMask)0  }
 };
 

src/debug.h

@@ -36,7 +36,8 @@ class Debug
                      Time         = 0x00000200,
                      ExtCmd       = 0x00000400,
                      Markdown     = 0x00000800,
-                     FilterOutput = 0x00001000
+                     FilterOutput = 0x00001000,
+                     Lex          = 0x00002000
                    };
     static void print(DebugMask mask,int prio,const char *fmt,...);
     static int  setFlag(const char *label);

src/declinfo.l

@@ -231,6 +231,7 @@ ID	"$"?([a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*)|(@[0-9]+)
 void parseFuncDecl(const QCString &decl,bool objC,QCString &cl,QCString &t,
                    QCString &n,QCString &a,QCString &ftl,QCString &exc)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, NULL);
   inputString   = decl;
   //printf("Input=`%s'\n",inputString);
   if (inputString==0) return;
@@ -307,6 +308,7 @@ void parseFuncDecl(const QCString &decl,bool objC,QCString &cl,QCString &t,
   //printf("type=`%s' class=`%s' name=`%s' args=`%s'\n",
   //        t.data(),cl.data(),n.data(),a.data());
 
+  printlex(yy_flex_debug, FALSE, __FILE__, NULL);
   return;
   
   

src/defargs.l

@@ -55,6 +55,7 @@
 #include "entry.h"
 #include "util.h"
 #include "arguments.h"
+#include "message.h"
   
 #define YY_NEVER_INTERACTIVE 1
 #define YY_NO_INPUT 1
@@ -527,6 +528,7 @@ void stringToArgumentList(const char *argsString,ArgumentList* al,QCString *extr
 {
   if (al==0) return; 
   if (argsString==0) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, NULL);
 
   g_copyArgValue=0;
   g_curArgDocs.resize(0);
@@ -549,6 +551,7 @@ void stringToArgumentList(const char *argsString,ArgumentList* al,QCString *extr
   defargsYYlex();
   if (extraTypeChars) *extraTypeChars=g_extraTypeChars;
   //printf("stringToArgumentList(%s) result=%s\n",argsString,argListToString(al).data());
+  printlex(yy_flex_debug, FALSE, __FILE__, NULL);
 }
 
 #if !defined(YY_FLEX_SUBMINOR_VERSION) 

src/doctokenizer.l

@@ -1189,6 +1189,7 @@ void doctokenizerYYFindSections(const char *input,Definition *d,
                                 MemberGroup *mg,const char *fileName)
 {
   if (input==0) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   g_inputString = input;
   //printf("parsing --->`%s'<---\n",input);
   g_inputPos    = 0;
@@ -1198,6 +1199,7 @@ void doctokenizerYYFindSections(const char *input,Definition *d,
   BEGIN(St_Sections);
   doctokenizerYYlineno = 1;
   doctokenizerYYlex();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void doctokenizerYYinit(const char *input,const char *fileName)

src/fortrancode.l

@@ -1115,6 +1115,7 @@ void parseFortranCode(CodeOutputInterface &od,const char *className,const QCStri
   (void)className;
 
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   g_code = &od;
   g_inputString   = s;
@@ -1174,6 +1175,7 @@ void parseFortranCode(CodeOutputInterface &od,const char *className,const QCStri
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 

src/fortranscanner.l

@@ -2339,7 +2339,12 @@ void FortranLanguageScanner::parseInput(const char *fileName,
                                         QStrList & /*filesInSameTranslationUnit*/)
 {
   g_thisParser = this;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
+
   ::parseMain(fileName,fileBuf,root);
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void FortranLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,

src/message.cpp

@@ -211,3 +211,31 @@ void err(const char *fmt, ...)
   vfprintf(warnFile, (QCString(error_str) + fmt).data(), args);
   va_end(args); 
 }
+
+void printlex(int dbg, bool enter, const char *lexName, const char *fileName)
+{
+  char *enter_txt = "entering";
+  char *enter_txt_uc = "Entering";
+
+  if (!enter)
+  {
+    enter_txt = "finished";
+    enter_txt_uc = "Finished";
+  }
+
+  if (dbg)
+  {
+    if (fileName)
+      fprintf(stderr,"--%s lexical analyzer: %s (for: %s)\n",enter_txt, lexName, fileName);
+    else
+      fprintf(stderr,"--%s lexical analyzer: %s\n",enter_txt, lexName);
+  }
+  else
+  {
+    if (fileName)
+      Debug::print(Debug::Lex,0,"%s lexical analyzer: %s (for: %s)\n",enter_txt_uc, lexName, fileName);
+    else
+      Debug::print(Debug::Lex,0,"%s lexical analyzer: %s\n",enter_txt_uc, lexName);
+  }
+
+}

src/message.h

@@ -29,4 +29,5 @@ extern void warn_uncond(const char *fmt, ...);
 extern void err(const char *fmt, ...);
 void initWarningFormat();
 
+extern void printlex(int dbg, bool enter, const char *lexName, const char *fileName);
 #endif

src/pre.l

@@ -2969,6 +2969,7 @@ void cleanUpPreprocessor()
 
 void preprocessFile(const char *fileName,BufStr &input,BufStr &output)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   uint orgOffset=output.curPos();
   //printf("##########################\n%s\n####################\n",
   //    input.data());
@@ -3177,6 +3178,7 @@ void preprocessFile(const char *fileName,BufStr &input,BufStr &output)
     }
   }
   DefineManager::instance().endContext();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void preFreeScanner()

src/pycode.l

@@ -1434,6 +1434,7 @@ void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
   
   //--------------------------------------
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   g_code = &od;
   g_inputString   = s;
@@ -1494,6 +1495,7 @@ void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 

src/pyscanner.l

@@ -1686,7 +1686,9 @@ void PythonLanguageScanner::parseInput(const char *fileName,
                                        QStrList & /*filesInSameTranslationUnit*/)
 {
   g_thisParser = this;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   ::parseMain(fileName,fileBuf,root);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 
   // May print the AST for debugging purposes
   // printAST(global_root);

src/scanner.l

@@ -6893,8 +6893,13 @@ void CLanguageScanner::parseInput(const char *fileName,
                                   QStrList & filesInSameTranslationUnit)
 {
   g_thisParser = this;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
+
   ::parseMain(fileName,fileBuf,root,
               sameTranslationUnit,filesInSameTranslationUnit);
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void CLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,

src/tclscanner.l

@@ -127,7 +127,7 @@ int TclFindElement(
     goto done;
   }
 
-  if (*p == '{') 
+  if (*p == '{')   /* } to keep vi happy */
   {
     openBraces = 1;
     p++;
@@ -2541,6 +2541,7 @@ tcl_inf("%s\n",fileName);
 
   tcl.input_string = input;
   if (tcl.input_string.length()<1) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
 
   msg("Parsing %s...\n",fileName);
   groupEnterFile(fileName,yylineno);
@@ -2554,6 +2555,7 @@ tcl_inf("%s\n",fileName);
   groupLeaveFile(tcl.file_name,yylineno);
   root->program.resize(0);
   myFile.close();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 //! Parse file and codify.
@@ -2583,6 +2585,7 @@ void TclLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,
   (void)collectXRefs;
 
   if (input.length()<1) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileDef ? fileDef->fileName().data(): NULL);
   tcl.input_string = input;
 
   QCString myNs="";
@@ -2655,6 +2658,7 @@ tcl_inf("%s (%d,%d) %d %d\n",myStr.ascii(),startLine,endLine,isExampleBlock,inli
   tcl.cl.clear();
   tcl.fn.clear();
   tcl.entry.clear();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileDef ? fileDef->fileName().data(): NULL);
 }
 
 bool TclLanguageScanner::needsPreprocessing(const QCString &extension)

src/vhdlcode.l

@@ -1517,6 +1517,7 @@ void parseVhdlCode(CodeOutputInterface &od,const char *className,const QCString
 {
   //printf("***parseCode() exBlock=%d exName=%s fd=%p\n",exBlock,exName,fd);
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   if (memberDef)
   {
@@ -1599,6 +1600,7 @@ void parseVhdlCode(CodeOutputInterface &od,const char *className,const QCString
     g_sourceFileDef=0;
   }
   g_startCode=FALSE;
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
 }
 
 void codeFreeVhdlScanner()

src/vhdlscanner.l

@@ -763,6 +763,7 @@ void VHDLLanguageScanner::parseInput(const char *fileName,
     VhdlDocGen::parseUCF(fileBuf,root,yyFileName,TRUE); 
     return; 
   }
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
 
   ::parserInit();
   yycont=getVhdlCont();
@@ -791,6 +792,7 @@ void VHDLLanguageScanner::parseInput(const char *fileName,
   free(lineIndex);
   inputFile.close();
   mapLibPackage(root);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void VHDLLanguageScanner::parseCode(CodeOutputInterface &codeOutIntf,