DSNLEXER now supports:

    1)  nested quotes.  This is in anticipation of broader usage of the
        file type/syntax.  A string like this in the file:
            "my ""favorate"" string"
        can be returned as
            my "favorite" string
    2)  CommentsAsTokens is implemented, so you can ask the lexer to return
        comments as tokens, so they can be preserved.  The default is to ignore
        them.  A comment is defined as any line that has # as its first
        non-blank character.  (This means comments cannot follow anything else
        on a line.)

CHANGELOG.txt

@@ -4,6 +4,21 @@ KiCad ChangeLog 2010
 Please add newer entries at the top, list the date and your name with
 email address.
 
+2010-Feb-20 UPDATE Dick Hollenbeck <dick@softplc.com>
+================================================================================
+++common
+    DSNLEXER now supports:
+    1)  nested quotes.  This is in anticipation of broader usage of the
+        file type/syntax.  A string like this in the file:
+            "my ""favorate"" string"
+        can be returned as
+            my "favorite" string
+    2)  CommentsAsTokens is implemented, so you can ask the lexer to return
+        comments as tokens, so they can be preserved.  The default is to ignore
+        them.  A comment is defined as any line that has # as its first
+        non-blank character.  (This means comments cannot follow anything else
+        on a line.)
+
 
 2010-Feb-19 UPDATE Jean-Pierre Charras <jean-pierre.charras@gipsa-lab.inpg.fr>
 ================================================================================

TODO.txt

@@ -96,10 +96,6 @@ P2) Write accessors for all items in PCB_VISIBLE such as grid control, so that
     Then example 2 in RS274xrevd_e.pdf will draw properly.
 
 
-Dick:
-D1) Get the nested quote support for DSNLEXER fixed up and committed.
-
-
 LAYER_WIDGET for PCBNEW (Dick)
 -----------------------
 L6) Test, and fix up any remaining issues with the PCB_VISIBLE support after P2)

common/dsnlexer.cpp

@@ -51,7 +51,6 @@ static int compare( const void* a1, const void* a2 )
 DSNLEXER::DSNLEXER( FILE* aFile, const wxString& aFilename,
     const KEYWORD* aKeywordTable, unsigned aKeywordCount ) :
         reader( aFile, 4096 )
-
 {
     keywords = aKeywordTable;
     keywordCount = aKeywordCount;
@@ -62,6 +61,8 @@ DSNLEXER::DSNLEXER( FILE* aFile, const wxString& aFilename,
 
     space_in_quoted_tokens = true;
 
+    commentsAreTokens = false;
+
     // "start" should never change until we change the reader.  The DSN
     // format spec supports an include file mechanism but we can add that later
     // using a std::stack to hold a stack of LINE_READERs to track nesting.
@@ -211,9 +212,23 @@ L_read:
             while( cur<limit && isSpace(*cur) )
                 ++cur;
 
-            // if the first non-blank character is #, this line is a comment.
+            // If the first non-blank character is #, this line is a comment.
+            // Comments cannot follow any other token on the same line.
             if( cur<limit && *cur=='#' )
-                goto L_read;
+            {
+                if( commentsAreTokens )
+                {
+                    // save the entire line, including new line as the current token.
+                    // the '#' character may not be at offset zero.
+                    curText = start;        // entire line is the token
+                    cur     = start;        // ensure a good curOffset below
+                    curTok  = DSN_COMMENT;
+                    head    = limit;        // do a readLine() on next call in here.
+                    goto exit;
+                }
+                else
+                    goto L_read;
+            }
         }
         else
         {
@@ -303,11 +318,55 @@ L_read:
         // a quoted string
         if( *cur == stringDelimiter )
         {
+            // New code, understands nested quotes, and deliberately restricts
+            // strings to a single line. Still strips off leading and trailing
+            // quotes, and now removes internal doubled up quotes
+#if 1
+            head = cur;
+
+            // copy the token, character by character so we can remove doubled up quotes.
+            curText.clear();
+
+            while( head < limit )
+            {
+                if( *head==stringDelimiter )
+                {
+                    if( head+1<limit && head[1]==stringDelimiter )
+                    {
+                        // include only one of the doubled-up stringDelimiters
+                        curText += *head;
+                        head    += 2;
+                        continue;
+                    }
+                    else if( head == cur )
+                    {
+                        ++head;     // skip the leading quote
+                        continue;
+                    }
+
+                    // fall thru
+                }
+
+                // check for a terminator
+                if( isStringTerminator( *head ) )
+                {
+                    curTok = DSN_STRING;
+                    ++head;
+                    goto exit;
+                }
+
+                curText += *head++;
+            }
+
+            wxString errtxt(_("Un-terminated delimited string") );
+            ThrowIOError( errtxt, CurOffset() );
+
+#else   // old code, did not understand nested quotes
             ++cur;  // skip over the leading delimiter: ",', or $
 
             head = cur;
 
-            while( head<limit &&  !isStringTerminator( *head ) )
+            while( head<limit  &&  !isStringTerminator( *head ) )
                 ++head;
 
             if( head >= limit )
@@ -323,6 +382,7 @@ L_read:
 
             curTok  = DSN_STRING;
             goto exit;
+#endif
         }
 
         // Maybe it is a token we will find in the token table.

include/dsnlexer.h

@@ -85,7 +85,7 @@ class DSNLEXER
     LINE_READER         reader;
     int                 stringDelimiter;
     bool                space_in_quoted_tokens; ///< blank spaces within quoted strings
-    bool                comments_are_tokens;    ///< true if should return comments as tokens
+    bool                commentsAreTokens;      ///< true if should return comments as tokens
 
     wxString            filename;
     int                 prevTok;        ///< curTok from previous NextTok() call.
@@ -194,8 +194,8 @@ public:
      */
     bool SetCommentsAreTokens( bool val )
     {
-        bool old = comments_are_tokens;
-        comments_are_tokens = val;
+        bool old = commentsAreTokens;
+        commentsAreTokens = val;
         return old;
     }