Implement "KiCad s-epression" support for boost::property_tree's ptree, the 8

bit string version of property_tree. The ram resident structure of the ptree is
mostly compatible with one created using the xml_parser from
boost::property_tree, with slight differences in the way atoms are stored. The
result is you can use Format() to convert from xml to s-expression, but not the
other way around. You can write a simple s-expression beautifier in just a few
lines of code.

The main value however is the s-expression parser, i.e. Scan(), which is an
alternative to crafting a custom recursive descent parser for a particular
grammar. The tipping point depends on whether you want to read only a small
portion of a much larger document. If so, then using the ptree will likely be a
"faster to code" route. Documentation on how to navigate a ptree can be found on
the boost website and there are a number of examples in the
pcbnew/eagle_plugin.cpp file in this project. Powerful path navigation support
makes it easy to extract a subset of a ptree.

common/CMakeLists.txt

@@ -71,6 +71,7 @@ set(COMMON_SRCS
     netlist_keywords.cpp
     newstroke_font.cpp
     projet_config.cpp
+    ptree.cpp
     richio.cpp
     selcolor.cpp
     string.cpp

common/ptree.cpp

+
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2013 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
+ * Copyright (C) 2013 KiCad Developers, see CHANGELOG.TXT for contributors.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you may find one here:
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+ * or you may search the http://www.gnu.org website for the version 2 license,
+ * or you may write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+#include <boost/property_tree/ptree.hpp>
+
+#include <assert.h>
+#include <macros.h>
+#include <ptree.h>
+
+typedef PTREE::const_iterator           CITER;
+typedef PTREE::iterator                 ITER;
+
+#if defined(DEBUG)
+ #define D(x)   x
+#else
+ #define D(x)
+#endif
+
+#define CTL_OMIT_NL                     (1<<0)
+#define CTL_IN_ATTRS                    (1<<1)
+
+
+//-----<Scan>------------------------------------------------------------------
+
+/**
+ * Function scanList
+ * reads a sexpr list from the input stream into a new node with key
+ * aLexer->CurText().
+ */
+inline void scanList( PTREE* aTree, DSNLEXER* aLexer )
+{
+    assert( aLexer->CurTok() == DSN_LEFT );
+
+    int tok = aLexer->NextTok();
+
+    const char* key = aLexer->CurText();
+
+    //D(printf( "%s: '%s'\n", __func__, key );)
+
+    PTREE* list = &aTree->push_back( PTREE::value_type( key, PTREE() ) )->second;
+
+    if( tok != DSN_RIGHT )
+    {
+        while( ( tok = aLexer->NextTok() ) != DSN_RIGHT )
+        {
+            if( tok == DSN_EOF )
+                aLexer->Unexpected( DSN_EOF );
+
+            Scan( list, aLexer );
+        }
+    }
+}
+
+
+inline void scanAtom( PTREE* aTree, DSNLEXER* aLexer )
+{
+    const char* key = aLexer->CurText();
+
+    //D(printf( "%s: '%s'\n", __func__, key );)
+
+    aTree->push_back( PTREE::value_type( key, PTREE() ) );
+}
+
+
+void Scan( PTREE* aTree, DSNLEXER* aLexer ) throw( IO_ERROR )
+{
+    int tok  = aLexer->CurTok();
+
+    // conditionally read first token.
+    if( tok == DSN_NONE )
+        tok = aLexer->NextTok();
+
+    if( tok == DSN_EOF )
+    {
+        aLexer->Unexpected( DSN_EOF );
+    }
+
+    if( tok == DSN_LEFT )
+    {
+        scanList( aTree, aLexer );
+    }
+    else
+    {
+        scanAtom( aTree, aLexer );
+    }
+}
+
+
+//-----<Format>------------------------------------------------------------------
+
+inline bool isAtom( CPTREE& aTree )
+{
+    return aTree.size()==0 && aTree.data().size()==0;
+}
+
+
+inline bool isLast( CPTREE& aTree, CITER it )
+{
+    CITER next = it;
+    ++next;
+    return next == aTree.end();
+}
+
+
+inline CITER next( CITER it )
+{
+    CITER n = it;
+    return ++n;
+}
+
+
+static void formatNode( OUTPUTFORMATTER* out, int aNestLevel, int aCtl,
+        const std::string& aKey, CPTREE& aTree ) throw( IO_ERROR );
+
+
+static void formatList( OUTPUTFORMATTER* out, int aNestLevel, int aCtl, CPTREE& aTree )
+    throw( IO_ERROR )
+{
+    for( CITER it = aTree.begin();  it != aTree.end();  ++it )
+    {
+        // Processing a tree which was read in with xml_parser?
+        if( it->first == "<xmlattr>" )
+        {
+            formatList( out, aNestLevel, aCtl | CTL_IN_ATTRS, it->second );
+            continue;
+        }
+
+        int ctl = 0;
+
+#if defined(DEBUG)
+        if( it->first == "field" )
+        {
+            int breakhere = 1;
+            (void) breakhere;
+        }
+#endif
+
+        if( isLast( aTree, it ) )   // is "it" the last one?
+        {
+            //if( !( aCtl & CTL_IN_ATTRS ) )
+                ctl = CTL_OMIT_NL;
+        }
+        else if( isAtom( next( it )->second ) )
+        {
+            /* if( !( aCtl & CTL_IN_ATTRS ) ) */
+                ctl = CTL_OMIT_NL;
+        }
+
+        formatNode( out, aNestLevel+1, ctl, it->first, it->second );
+    }
+}
+
+
+static void formatNode( OUTPUTFORMATTER* out, int aNestLevel, int aCtl,
+        const std::string& aKey, CPTREE& aTree )
+    throw( IO_ERROR )
+{
+    if( !isAtom( aTree ) )     // is a list, not an atom
+    {
+        int ctl = CTL_OMIT_NL;
+
+        // aTree is list and its first child is a list
+        if( aTree.size() && !isAtom( aTree.begin()->second ) && !aTree.data().size() )
+            ctl = 0;
+
+        out->Print( aNestLevel, "(%s%s", out->Quotes( aKey ).c_str(), ctl & CTL_OMIT_NL ? "" : "\n" );
+
+        if( aTree.data().size() )   // only xml typically uses "data()", not sexpr.
+        {
+            out->Print( 0, " %s%s",
+                out->Quotes( aTree.data() ).c_str(),
+                aTree.size() ? "\n" : ""
+                );
+        }
+
+        formatList( out, aNestLevel, aCtl, aTree );
+
+        out->Print( 0, ")%s", aCtl & CTL_OMIT_NL ? "" : "\n" );
+    }
+
+    else            // is an atom, not a list
+    {
+        const char* atom = out->Quotes( aKey ).c_str();
+        out->Print( 0, " %s", atom );
+    }
+}
+
+
+void Format( OUTPUTFORMATTER* out, int aNestLevel, int aCtl, CPTREE& aTree ) throw( IO_ERROR )
+{
+    if( aTree.size() == 1 && !aTree.data().size() )
+    {
+        // The topmost node is basically only a container for the document root.
+        // It anchors the paths which traverse the tree deeper.
+        CITER   it = aTree.begin();
+        formatNode( out, aNestLevel, aCtl, it->first, it->second );
+    }
+    else
+    {
+        // This is not expected, neither for sexpr nor xml.
+        formatNode( out, aNestLevel, aCtl, "", aTree );
+    }
+}
+

include/ptree.h

+#ifndef PTREE_H_
+#define PTREE_H_
+
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2013 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
+ * Copyright (C) 2013 KiCad Developers, see CHANGELOG.TXT for contributors.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you may find one here:
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+ * or you may search the http://www.gnu.org website for the version 2 license,
+ * or you may write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+
+/*
+
+Implement "KiCad s-epression" support for boost::property_tree's ptree, the 8
+bit string version of property_tree. The ram resident structure of the ptree is
+mostly compatible with one created using the xml_parser from
+boost::property_tree, with slight differences in the way atoms are stored. The
+result is you can use Format() to convert from xml to s-expression, but not the
+other way around. You can write a simple s-expression beautifier in just a few
+lines of code.
+
+The main value however is the s-expression parser, i.e. Scan(), which is an
+alternative to crafting a custom recursive descent parser for a particular
+grammar. The tipping point depends on whether you want to read only a small
+portion of a much larger document. If so, then using the ptree will likely be a
+"faster to code" route. Documentation on how to navigate a ptree can be found on
+the boost website and there are a number of examples in the
+pcbnew/eagle_plugin.cpp file in this project. Powerful path navigation support
+makes it easy to extract a subset of a ptree.
+
+*/
+
+
+#include <boost/property_tree/ptree_fwd.hpp>
+#include <richio.h>
+#include <dsnlexer.h>
+
+typedef boost::property_tree::ptree     PTREE;
+typedef const PTREE                     CPTREE;
+
+
+/**
+ * Function Scan
+ * fills an empty PTREE with information from a KiCad s-expresion stream.  Use
+ * a DSNLEXER with an empty keyword table as @a aLexer.  Useful for parsing
+ * s-expression files or strings of arbitrary grammars, say from a file or clipboard.
+ * The s-expression must be "KiCad compatible".  See Documentation/s-expressions.txt
+ * for this KiCad compatible definition (it is the non-specctra mode).
+ * And also see in tools/property_tree.cpp for example usage.
+ *
+ * <code>
+ *
+ * FILE* fp = fopen( argv[1], "r" );
+ *
+ * static const KEYWORD empty_keywords[1] = {};
+ *
+ * DSNLEXER   lexer( empty_keywords, 0, fp, wxString( FROM_UTF8( argv[1] ) ) );
+ *
+ * try
+ * {
+ *     PTREE   doc;
+ *     Scan( &doc, &lexer );
+ * }
+ * catch( IO_ERROR ioe )
+ * {
+ *     fprintf( stderr, "%s\n", TO_UTF8( ioe.errorText ) );
+ * }
+ *
+ * </code>
+ */
+void Scan( PTREE* aTree, DSNLEXER* aLexer ) throw ( IO_ERROR );
+
+/**
+ * Function Format
+ * outputs a PTREE into s-expression format via an OUTPUTFORMATTER derivative.
+ */
+void Format( OUTPUTFORMATTER* out, int aNestLevel, int aCtl, CPTREE& aTree ) throw( IO_ERROR );
+
+#endif  // PTREE_H_

tools/CMakeLists.txt

@@ -65,13 +65,14 @@ add_executable( test-nm-biu-to-ascii-mm-round-tripping
     test-nm-biu-to-ascii-mm-round-tripping.cpp
     )
 
-add_executable( parser_gen
+add_executable( property_tree
     EXCLUDE_FROM_ALL
-    parser_gen.cpp
+    property_tree.cpp
     ../common/richio.cpp
     ../common/dsnlexer.cpp
+    ../common/ptree.cpp
     )
-target_link_libraries( parser_gen
+target_link_libraries( property_tree
     ${wxWidgets_LIBRARIES}
     )
 

tools/parser_gen.cpp

-/*
- * This program source code file is part of KiCad, a free EDA CAD application.
- *
- * Copyright (C) 2012 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
- * Copyright (C) 2012 KiCad Developers, see CHANGELOG.TXT for contributors.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, you may find one here:
- * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
- * or you may search the http://www.gnu.org website for the version 2 license,
- * or you may write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
- */
-
-
-// This is wanting to be an s-expression C++ parser generator.  Feed it a sample
-// file and maybe someday it will generate a C++ file which uses DSNLEXER to
-// parse the described grammar OK.
-
-// Until then, it is a non-specctra mode s-expression beautifier.
-
-
-#include <assert.h>
-#include <richio.h>
-#include <dsnlexer.h>
-#include <macros.h>
-#include <boost/ptr_container/ptr_vector.hpp>
-
-
-// http://sexpr.sourceforge.net/  see comments about graphviz
-// http://www.codeproject.com/KB/recipes/JSON_Spirit.aspx
-
-#define D(x)   x
-//#define D(x)
-
-
-/**
- * Class ELEM
- */
-class ELEM
-{
-protected:
-    int             token;
-    std::string     text;
-
-    typedef boost::ptr_vector<ELEM> ELEMS;
-    typedef ELEMS::const_iterator   ELEMS_CITER;
-    typedef ELEMS::iterator         ELEMS_ITER;
-
-    ELEMS           kids;      ///< ELEM pointers
-
-public:
-
-    // there are two constructors, one for a list, one for an atom
-
-    /// List constructor
-    ELEM( int aToken ) :
-        token( aToken ),
-        text( "" )
-    {
-        // D( printf( "ELEM%p: list\n", this ); )
-    }
-
-    /// Atom constructor
-    ELEM( const std::string& aText, int aToken ) :
-        token( aToken ),
-        text( aText )
-    {
-        // D( printf( "ELEM%p: '%s'\n", this, text.c_str() ); )
-    }
-
-    int  Token() const { return token; }
-
-    const char* Text() { return text.c_str(); }
-
-
-    /**
-     * Function Format
-     * writes this object as ASCII out to an OUTPUTFORMATTER
-     * @param out The formatter to write to.
-     * @param nestLevel A multiple of the number of spaces to preceed the output with.
-     * @throw IO_ERROR if a system error writing the output, such as a full disk.
-     */
-    void Format( OUTPUTFORMATTER* aFormatter, int aNestLevel = 0, int aControlBits = 0 );
-
-#define CTL_OMIT_NL         (1<<0)
-
-    /**
-     * Function Length
-     * returns the number of ELEMs in this ELEM.
-     * @return int - the count of children
-     */
-    int     Length() const
-    {
-        return kids.size();
-    }
-
-    void    Append( ELEM* aElem )
-    {
-        kids.push_back( aElem );
-    }
-
-    ELEM*   Replace( int aIndex, ELEM* aElem )
-    {
-        ELEMS::auto_type ret = kids.replace( aIndex, aElem );
-        return ret.release();
-    }
-
-    ELEM*   Remove( int aIndex )
-    {
-        ELEMS::auto_type ret = kids.release( kids.begin()+aIndex );
-        return ret.release();
-    }
-
-    void    Insert( int aIndex, ELEM* aElem )
-    {
-        kids.insert( kids.begin()+aIndex, aElem );
-    }
-
-    ELEM* At( int aIndex ) const
-    {
-        const ELEM&  ref = kids.at( aIndex );
-        return (ELEM*) &ref;
-    }
-
-    ELEM* operator[]( int aIndex ) const
-    {
-        return At( aIndex );
-    }
-
-    void    Delete( int aIndex )
-    {
-        kids.erase( kids.begin()+aIndex );
-    }
-};
-
-
-void ELEM::Format( OUTPUTFORMATTER* out, int nestLevel, int ctl )
-{
-    if( token == DSN_LEFT )     // this is a list
-    {
-        out->Print( nestLevel, "(" );
-
-        const int count = Length();
-        for( int i=0; i<count;  ++i )
-        {
-            ELEM* cur  = At( i );
-            ELEM* next = i < count-1 ? At( i+1 ) : NULL;
-
-            if( i > 0 )
-                out->Print( 0, " " );
-
-            if( next && next->token == DSN_LEFT )
-            {
-                cur->Format( out, nestLevel+1, 0 );
-            }
-            else
-            {
-                cur->Format( out, nestLevel+1, CTL_OMIT_NL );
-            }
-        }
-
-        out->Print( 0, ")%s", ctl & CTL_OMIT_NL ? "" : "\n" );
-    }
-    else    // this is an atom
-    {
-        const char* s = out->Quotes( text ).c_str();
-        out->Print( 0, "%s%s", s, ctl & CTL_OMIT_NL ? "" : "\n" );
-    }
-}
-
-ELEM* Scan( DSNLEXER* lex );
-ELEM* ScanList( DSNLEXER* lex );
-ELEM* ScanAtom( DSNLEXER* lex );
-
-
-void usage()
-{
-    fprintf( stderr, "Usage: parser_gen <grammar_s-expression_file>\n" );
-    exit( 1 );
-}
-
-
-static KEYWORD empty_keywords[1] = {};
-
-
-ELEM* Scan( DSNLEXER* lex )
-{
-    ELEM*   elem = NULL;
-    int     tok  = lex->CurTok();
-
-    // conditionally read first token.
-    if( tok == DSN_NONE )
-        tok = lex->NextTok();
-
-    if( tok == DSN_EOF )
-    {
-        lex->Unexpected( DSN_EOF );
-    }
-
-    if( tok == DSN_LEFT )
-    {
-        elem = ScanList( lex );
-    }
-    else
-    {
-        elem = ScanAtom( lex );
-    }
-
-    return elem;
-}
-
-
-/**
- * Function ScanList
- * reads and returns a sexpList from the input stream.
- */
-ELEM* ScanList( DSNLEXER* lex )
-{
-    int     tok;
-    ELEM*   list = NULL;
-
-    assert( lex->CurTok() == DSN_LEFT );
-
-    list = new ELEM( DSN_LEFT );
-
-    while( ( tok = lex->NextTok() ) != DSN_RIGHT )
-    {
-        if( tok == DSN_EOF )
-            lex->Unexpected( DSN_EOF );
-
-        ELEM* elem = Scan( lex );
-        list->Append( elem );
-    }
-
-    return list;
-}
-
-
-ELEM* ScanAtom( DSNLEXER* lex )
-{
-    return new ELEM( lex->CurText(), lex->CurTok() );
-}
-
-
-int main( int argc, char** argv )
-{
-    if( argc != 2 )
-    {
-        usage();
-    }
-
-    FILE*   fp = fopen( argv[1], "rt" );
-    if( !fp )
-    {
-        fprintf( stderr, "Unable to open '%s'\n", argv[1] );
-        usage();
-    }
-
-    DSNLEXER   lexer( empty_keywords, 0, fp, wxString( FROM_UTF8( argv[1] ) ) );
-
-    try
-    {
-        ELEM* elem = Scan( &lexer );
-
-        if( elem )
-        {
-            STRING_FORMATTER sf;
-
-            elem->Format( &sf, 0 );
-
-            printf( "%s", sf.GetString().c_str() );
-        }
-    }
-    catch( IO_ERROR ioe )
-    {
-        fprintf( stderr, "%s\n", TO_UTF8( ioe.errorText ) );
-    }
-}
-

tools/property_tree.cpp

+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2012 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
+ * Copyright (C) 2012 KiCad Developers, see CHANGELOG.TXT for contributors.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you may find one here:
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+ * or you may search the http://www.gnu.org website for the version 2 license,
+ * or you may write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+
+// This is a propertytree test utility.
+// It can convert XML to sexpressions or beautify s-expressions in non-specctra mode.
+
+
+#include <assert.h>
+#include <ptree.h>
+#include <richio.h>
+#include <dsnlexer.h>
+#include <macros.h>
+#include <boost/property_tree/xml_parser.hpp>
+
+
+void usage()
+{
+    fprintf( stderr, "Usage: parser_gen <grammar_s-expression_file>\n" );
+    exit( 1 );
+}
+
+
+int main( int argc, char** argv )
+{
+    if( argc != 2 )
+    {
+        usage();
+    }
+
+    FILE*   fp = fopen( argv[1], "r" );
+    if( !fp )
+    {
+        fprintf( stderr, "Unable to open '%s'\n", argv[1] );
+        usage();
+    }
+
+    static const KEYWORD empty_keywords[1] = {};
+
+    DSNLEXER   lexer( empty_keywords, 0, fp, wxString( FROM_UTF8( argv[1] ) ) );
+
+    try
+    {
+        PTREE   doc;
+
+#if 0
+        using namespace boost::property_tree;
+
+        read_xml( argv[1], doc, xml_parser::trim_whitespace | xml_parser::no_comments );
+#else
+        Scan( &doc, &lexer );
+#endif
+
+#if 1
+        STRING_FORMATTER sf;
+        Format( &sf, 0, 0, doc );
+        printf( "%s", sf.GetString().c_str() );
+#else
+        // writing the unchanged ptree in file2.xml
+        boost::property_tree::xml_writer_settings<char> settings( ' ', 2 );
+        write_xml( "/tmp/output.xml", doc, std::locale(), settings );
+#endif
+
+    }
+    catch( IO_ERROR ioe )
+    {
+        fprintf( stderr, "%s\n", TO_UTF8( ioe.errorText ) );
+    }
+}
+