Clarify, extend, and rename ky*.py to kicad_netlist_reader.py.

Significantly enhance the bom_csv_grouped_by_value.py BOM generator.
IMO it at this moment, the best BOM production tool for KiCad.

scripts/bom-in-python/bom-generation.py

@@ -4,8 +4,10 @@
 # Example: Sorted and Grouped HTML BOM with more advanced grouping
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import sys
 
 # Start with a basic html template
@@ -43,9 +45,9 @@ def myEqu(self, other):
     result = True
     if self.getValue() != other.getValue():
         result = False
-    elif self.getLib() != other.getLib():
+    elif self.getLibName() != other.getLibName():
         result = False
-    elif self.getPart() != other.getPart():
+    elif self.getPartName() != other.getPartName():
         result = False
     elif self.getFootprint() != other.getFootprint():
         result = False
@@ -61,18 +63,18 @@ def myEqu(self, other):
 # Override the component equivalence operator - it is important to do this
 # before loading the netlist, otherwise all components will have the original
 # equivalency operator.
-ky_generic_netlist_reader.component.__equ__ = myEqu
+kicad_netlist_reader.comp.__equ__ = myEqu
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # video.tmp. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write too, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
 # Output a set of rows for a header providing general information
@@ -88,9 +90,11 @@ row += "<th>Description</th>" + "<th>Vendor</th></tr>"
 
 html = html.replace('<!--TABLEROW-->', row + "<!--TABLEROW-->")
 
+components = net.getInterestingComponents()
+
 # Get all of the components in groups of matching parts + values
-# (see ky_generic_netlist_reader.py)
-grouped = net.groupComponents()
+# (see kicad_netlist_reader.py)
+grouped = net.groupComponents(components)
 
 # Output all of the component information
 for group in grouped:
@@ -99,17 +103,19 @@ for group in grouped:
     # Add the reference of every component in the group and keep a reference
     # to the component so that the other data can be filled in once per group
     for component in group:
-        refs += component.getRef() + ", "
+        if len(refs) > 0:
+            refs += ", "
+        refs += component.getRef()
         c = component
 
     row = "\n    "
     row += "<tr><td>" + refs +"</td><td>" + str(len(group))
-    row += "</td><td>" + c.getValue() + "</td><td>" + c.getLib() + "/"
-    row += c.getPart() + "</td><td>" + c.getDatasheet() + "</td><td>"
+    row += "</td><td>" + c.getValue() + "</td><td>" + c.getLibName() + ":"
+    row += c.getPartName() + "</td><td>" + c.getDatasheet() + "</td><td>"
     row += c.getDescription() + "</td><td>" + c.getField("Vendor")
     row += "</td></tr>"
 
     html = html.replace('<!--TABLEROW-->', row + "<!--TABLEROW-->")
 
 # Print the formatted html to output file
-print >> f, html
+print(html, file=f)

scripts/bom-in-python/bom_csv_by_ref.py

@@ -4,21 +4,23 @@
 # Example: Tab delimited list (The same as std output) Ungrouped
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import csv
 import sys
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # the command line option. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write to, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
 # Create a new csv writer object to use as the output formatter, although we
@@ -32,7 +34,9 @@ out.writerow(['Tool:', net.getTool()])
 out.writerow(['Component Count:', len(net.components)])
 out.writerow(['Ref', 'Value', 'Part', 'Documentation', 'Description', 'Vendor'])
 
+components = net.getInterestingComponents()
+
 # Output all of the component information
-for c in net.components:
-    out.writerow([c.getRef(), c.getValue(), c.getLib() + "/" + c.getPart(),
+for c in components:
+    out.writerow([c.getRef(), c.getValue(), c.getLibName() + ":" + c.getPartName(),
         c.getDatasheet(), c.getDescription(), c.getField("Vendor")])

scripts/bom-in-python/bom_csv_by_ref_v2.py

@@ -4,21 +4,23 @@
 # Example: Ungrouped (One component per row) CSV output
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import csv
 import sys
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # the command line option. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write to, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
 # Create a new csv writer object to use as the output formatter
@@ -31,8 +33,10 @@ out.writerow(['Tool:', net.getTool()])
 out.writerow(['Component Count:', len(net.components)])
 out.writerow(['Ref', 'Value', 'Footprint', 'Datasheet', 'Manufacturer', 'Vendor'])
 
+components = net.getInterestingComponents()
+
 # Output all of the component information (One component per row)
-for c in net.components:
+for c in components:
     out.writerow([c.getRef(), c.getValue(), c.getFootprint(), c.getDatasheet(),
         c.getField("Manufacturer"), c.getField("Vendor")])
 

scripts/bom-in-python/bom_csv_grouped_by_value.py

@@ -4,49 +4,123 @@
 # Example: Sorted and Grouped CSV BOM
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import csv
 import sys
 
+
+if len(sys.argv) != 3:
+    print("Usage ", __file__, "<generic_netlist.xml> <output.csv>", file=sys.stderr)
+    sys.exit(1)
+
+
 # Generate an instance of a generic netlist, and load the netlist tree from
 # the command line option. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write to, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
+# subset the components to those wanted in the BOM, controlled
+# by <configure> block in kicad_netlist_reader.py
+components = net.getInterestingComponents()
+
+compfields = net.gatherComponentFieldUnion(components)
+partfields = net.gatherLibPartFieldUnion()
+
+# remove Reference, Value, Datasheet, and Footprint, they will come from 'columns' below
+partfields -= set( ['Reference', 'Value', 'Datasheet', 'Footprint'] )
+
+columnset = compfields | partfields     # union
+
+# prepend an initial 'hard coded' list and put the enchillada into list 'columns'
+columns = ['Item', 'Qty', 'Reference(s)', 'Value', 'LibPart', 'Footprint', 'Datasheet'] + sorted(list(columnset))
+
 # Create a new csv writer object to use as the output formatter
-out = csv.writer(f, lineterminator='\n', delimiter=',', quotechar='\"', quoting=csv.QUOTE_ALL)
+out = csv.writer(f, lineterminator='\n', delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)
 
-# Output a set of rows for a header providing general information
+# Output a set of rows as a header providing general information
 out.writerow(['Source:', net.getSource()])
 out.writerow(['Date:', net.getDate()])
 out.writerow(['Tool:', net.getTool()])
-out.writerow(['Component Count:', len(net.components)])
-out.writerow(['Ref', 'Qnty', 'Value', 'Part', 'Datasheet', 'Description', 'Vendor'])
+out.writerow(['Component Count:', len(components)])
+out.writerow([])
+out.writerow(['Individual Components:'])
+out.writerow([])                        # blank line
+out.writerow(columns)
+
+# Output all the interesting components individually first:
+row = []
+for c in components:
+    del row[:]
+    row.append('')                                      # item is blank in individual table
+    row.append('')                                      # Qty is always 1, why print it
+    row.append( c.getRef() )                            # Reference
+    row.append( c.getValue() )                          # Value
+    row.append( c.getLibName() + ":" + c.getPartName() ) # LibPart
+    #row.append( c.getDescription() )
+    row.append( c.getFootprint() )
+    row.append( c.getDatasheet() )
+
+    # from column 7 upwards, use the fieldnames to grab the data
+    for field in columns[7:]:
+        row.append( c.getField( field ) );
+
+    out.writerow(row)
+
+
+out.writerow([])                        # blank line
+out.writerow([])                        # blank line
+out.writerow([])                        # blank line
+
+out.writerow(['Collated Components:'])
+out.writerow([])                        # blank line
+out.writerow(columns)                   # reuse same columns
+
+
 
 # Get all of the components in groups of matching parts + values
-# (see ky_generic_netlist_reader.py)
-grouped = net.groupComponents()
+# (see kicad_netlist_reader.py)
+grouped = net.groupComponents(components)
 
-# Output all of the component information
+
+# Output component information organized by group, aka as collated:
+item = 0
 for group in grouped:
+    del row[:]
     refs = ""
 
     # Add the reference of every component in the group and keep a reference
     # to the component so that the other data can be filled in once per group
     for component in group:
-        refs += component.getRef() + ", "
+        if len(refs) > 0:
+            refs += ", "
+        refs += component.getRef()
         c = component
 
     # Fill in the component groups common data
-    out.writerow([refs, len(group), c.getValue(), c.getLib() + "/" + c.getPart(), c.getDatasheet(),
-        c.getDescription(), c.getField("Vendor")])
+    # columns = ['Item', 'Qty', 'Reference(s)', 'Value', 'LibPart', 'Footprint', 'Datasheet'] + sorted(list(columnset))
+    item += 1
+    row.append( item )
+    row.append( len(group) )
+    row.append( refs );
+    row.append( c.getValue() )
+    row.append( c.getLibName() + ":" + c.getPartName() )
+    row.append( net.getGroupFootprint(group) )
+    row.append( net.getGroupDatasheet(group) )
+
+    # from column 7 upwards, use the fieldnames to grab the data
+    for field in columns[7:]:
+        row.append( net.getGroupField(group, field) );
 
+    out.writerow( row )
 
+f.close()

scripts/bom-in-python/bom_html_by_value.py

@@ -4,8 +4,10 @@
 # Example: Sorted and Grouped HTML BOM with more advanced grouping
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import sys
 
 # Start with a basic html template
@@ -43,9 +45,9 @@ def myEqu(self, other):
     result = True
     if self.getValue() != other.getValue():
         result = False
-    elif self.getLib() != other.getLib():
+    elif self.getLibName() != other.getLibName():
         result = False
-    elif self.getPart() != other.getPart():
+    elif self.getPartName() != other.getPartName():
         result = False
     elif self.getFootprint() != other.getFootprint():
         result = False
@@ -61,18 +63,18 @@ def myEqu(self, other):
 # Override the component equivalence operator - it is important to do this
 # before loading the netlist, otherwise all components will have the original
 # equivalency operator.
-ky_generic_netlist_reader.component.__equ__ = myEqu
+kicad_netlist_reader.comp.__equ__ = myEqu
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # video.xml. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write too, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
 # Output a set of rows for a header providing general information
@@ -90,9 +92,11 @@ row += "<th>PartNumber</th>" + "<th>Vendor</th></tr>"
 
 html = html.replace('<!--TABLEROW-->', row + "<!--TABLEROW-->")
 
+components = net.getInterestingComponents()
+
 # Get all of the components in groups of matching parts + values
-# (see ky_generic_netlist_reader.py)
-grouped = net.groupComponents()
+# (see kicad_netlist_reader.py)
+grouped = net.groupComponents(components)
 
 # Output all of the component information
 for group in grouped:
@@ -101,12 +105,14 @@ for group in grouped:
     # Add the reference of every component in the group and keep a reference
     # to the component so that the other data can be filled in once per group
     for component in group:
-        refs += component.getRef() + ", "
+        if len(refs) > 0:
+            refs += ", "
+        refs += component.getRef()
         c = component
 
     row = "<tr><td>" + refs +"</td><td>" + str(len(group))
     row += "</td><td>" + c.getValue() + "</td><td>"
-    row += c.getLib() + "/" + c.getPart() + "</td><td>"
+    row += c.getLibName() + ":" + c.getPartName() + "</td><td>"
     #row += c.getDatasheet() + "</td><td>"
     row += c.getDescription() + "</td><td>"
     row += c.getField("PartNumber") + "</td><td>"
@@ -116,4 +122,4 @@ for group in grouped:
     html = html.replace('<!--TABLEROW-->', row + "<!--TABLEROW-->")
 
 # Print the formatted html to output file
-print >> f, html
+print(html, file=f)

scripts/bom-in-python/bom_html_grouped_by_value.py

@@ -4,8 +4,10 @@
 # Example: Sorted and Grouped HTML BOM
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import sys
 
 # Start with a basic html template
@@ -30,14 +32,14 @@ html = """
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # the command line option. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write to, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
 # Output a set of rows for a header providing general information
@@ -53,9 +55,11 @@ row += "<th>Description</th>" + "<th>Vendor</th></tr>"
 
 html = html.replace('<!--TABLEROW-->', row + "<!--TABLEROW-->")
 
+components = net.getInterestingComponents()
+
 # Get all of the components in groups of matching parts + values
-# (see ky_generic_netlist_reader.py)
-grouped = net.groupComponents()
+# (see kicad_netlist_reader.py)
+grouped = net.groupComponents(components)
 
 # Output all of the component information
 for group in grouped:
@@ -64,16 +68,18 @@ for group in grouped:
     # Add the reference of every component in the group and keep a reference
     # to the component so that the other data can be filled in once per group
     for component in group:
-        refs += component.getRef() + ", "
+        if len(refs) > 0:
+            refs += ", "
+        refs += component.getRef()
         c = component
 
     row = "<tr><td>" + refs +"</td><td>" + str(len(group))
-    row += "</td><td>" + c.getValue() + "</td><td>" + c.getLib() + "/"
-    row += c.getPart() + "</td><td>" + c.getDatasheet() + "</td><td>"
+    row += "</td><td>" + c.getValue() + "</td><td>" + c.getLibName() + ":"
+    row += c.getPartName() + "</td><td>" + c.getDatasheet() + "</td><td>"
     row += c.getDescription() + "</td><td>" + c.getField("Vendor")
     row += "</td></tr>"
 
     html = html.replace('<!--TABLEROW-->', row + "<!--TABLEROW-->")
 
 # Print the formatted html to the file
-print >> f, html
+print(html, file=f)

scripts/bom-in-python/ky_generic_netlist_reader.py → scripts/bom-in-python/kicad_netlist_reader.py

@@ -7,99 +7,64 @@
 # is due to be deprecated in 3.0+ soon
 #
 
+from __future__ import print_function
 import sys
 import xml.sax as sax
+import re
+import pdb
 
+#-----<Configure>----------------------------------------------------------------
 
-class component():
-    """Class for a set of component information"""
-    def __init__(self, element):
-        self.element = element      
-        self.libpart = None
-        
-        # Set to true when this component is included in a component group
-        self.grouped = False
-    
-    def __eq__(self, other):
-        """Equlivalency operator, remember this can be easily overloaded"""
-        result = False
-        if self.getValue() == other.getValue():
-            if self.getLib() == other.getLib():
-                if self.getPart() == other.getPart():
-                    result = True
-        return result
+# excluded_fields is a list of regular expressions.  If any one matches a field
+# from either a component or a libpart, then that will not be included as a
+# column in the BOM.  Otherwise all columns from all used libparts and components
+# will be unionized and will appear.  Some fields are impossible to blacklist, such
+# as Ref, Value, Footprint, and Datasheet.  Additionally Qty and Item are supplied
+# unconditionally as columns, and may not be removed.
+excluded_fields = [
+    #'Price@1000'
+    ]
 
-    def setPart(self, part):
-        self.libpart = part
 
-    def setValue(self, value):
-        """Set the value of this component"""
-        v = self.element.getChild("value")
-        if v:
-            v.setChars(value)
-        
-    def getValue(self):
-        return self.element.get("value")
+# You may exlude components from the BOM by either:
+#
+# 1) adding a custom field named "Installed" to your components and filling it
+# with a value of "NU" (Normally Uninstalled).
+# See netlist.getInterestingComponents(), or
+#
+# 2) blacklisting it in any of the three following lists:
 
-    def getRef(self):
-        return self.element.get("comp", "ref")
 
-    def getFootprint(self):
-        return self.element.get("footprint")        
+# regular expressions which match component 'Reference' fields of components that
+# are to be excluded from the BOM.
+excluded_references = [
+    'TP[0-9]+'              # all test points
+    ]
 
-    def getDatasheet(self):
-        return self.element.get("datasheet")
 
-    def getLib(self):
-        return self.element.get("libsource", "lib")
-        
-    def getPart(self):
-        return self.element.get("libsource", "part") 
-    
-    def getTimestamp(self):
-        return self.element.get("tstamp")
-        
-    def getDescription(self):
-        # When attempting to access the part, we must take care in case the part
-        # cannot be found in the netlist
-        try:
-            d = self.libpart.getDescription()
-        except AttributeError:
-            d = ""
-        return d
+# regular expressions which match component 'Value' fields of components that
+# are to be excluded from the BOM.
+excluded_values = [
+    'MOUNTHOLE',
+    'SCOPETEST',
+    'MOUNT_HOLE',
+    'SOLDER_BRIDGE.*'
+    ]
 
-    def getDatasheet(self):
-        # When attempting to access the part, we must take care in case the part
-        # cannot be found in the netlist
-        try:
-            d = self.libpart.getDatasheet()
-        except AttributeError:
-            d = ""
-        return d
-    
-    def getField(self, name):
-        """Return the value of a field named name. The component is first 
-        checked for the field, and then the components library part is checked
-        for the field. If the field doesn't exist in either, an empty string is
-        returned
-        
-        Keywords:
-        name -- The name of the field to return the value for
-        
-        """
-        field = self.element.get("field", "name", name)
-        if field == "":
-            try:
-                field = self.libpart.getField(name)
-            except AttributeError:
-                field = ""
-        return field
 
+# regular expressions which match component 'Footprint' fields of components that
+# are to be excluded from the BOM.
+excluded_footprints = [
+    #'MOUNTHOLE'
+    ]
+
+#-----</Configure>---------------------------------------------------------------
 
-class netlistElement():
-    """Generic netlist element. All elements for a netlist tree which can be 
+
+class xmlElement():
+    """xml element which can represent all nodes of the netlist tree.  It can be
     used to easily generate various output formats by propogating format
-    requests to all children
+    requests to children recursively.
     """
     def __init__(self, name, parent=None):
         self.name = name
@@ -107,55 +72,57 @@ class netlistElement():
         self.parent = parent
         self.chars = ""
         self.children = []
-        self.indent = ""
-            
+
     def __str__(self):
         """String representation of this netlist element
-        
+
         """
-        return (self.name + "[" + self.chars + "]" + " attr:" + 
-            str(len(self.attributes[a])))            
-    
-    def formatXML(self, amChild=False):
+        return self.name + "[" + self.chars + "]" + " attr_count:" + str(len(self.attributes))
+
+    def formatXML(self, nestLevel=0, amChild=False):
         """Return this element formatted as XML
-        
+
         Keywords:
-        amChild -- If set to True, the start of document is not returned
-        
+        nestLevel -- increases by one for each level of nesting.
+        amChild -- If set to True, the start of document is not returned.
+
         """
         s = ""
 
+        indent = ""
+        for i in range(nestLevel):
+            indent += "    "
+
         if not amChild:
-            s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" 
-        
-        s += self.indent + "<" + self.name
+            s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+
+        s += indent + "<" + self.name
         for a in self.attributes:
             s += " " + a + "=\"" + self.attributes[a] + "\""
 
         if (len(self.chars) == 0) and (len(self.children) == 0):
             s += "/>"
         else:
-            s += ">" + self.chars                                                        
-        
+            s += ">" + self.chars
+
         for c in self.children:
-            c.indent += self.indent + "    "
             s += "\n"
-            s += c.formatXML(True)
-                    
+            s += c.formatXML(nestLevel+1, True)
+
         if (len(self.children) > 0):
-            s += "\n" + self.indent        
-         
-        if (len(self.children) > 0) or (len(self.chars) > 0): 
-            s += "</" + self.name + ">"        
-                  
+            s += "\n" + indent
+
+        if (len(self.children) > 0) or (len(self.chars) > 0):
+            s += "</" + self.name + ">"
+
         return s
 
     def formatHTML(self, amChild=False):
         """Return this element formatted as HTML
-        
+
         Keywords:
         amChild -- If set to True, the start of document is not returned
-        
+
         """
         s = ""
 
@@ -169,125 +136,296 @@ class netlistElement():
                 </head>
                 <body>
                 <table>
-                """ 
+                """
 
         s += "<tr><td><b>" + self.name + "</b><br>" + self.chars + "</td><td><ul>"
         for a in self.attributes:
             s += "<li>" + a + " = " + self.attributes[a] + "</li>"
 
         s += "</ul></td></tr>\n"
-                                                                
+
         for c in self.children:
-            s += c.formatHTML(True)        
-         
+            s += c.formatHTML(True)
+
         if not amChild:
             s += """</table>
                 </body>
-                </html>"""        
-                  
-        return s    
-                     
+                </html>"""
+
+        return s
+
     def addAttribute(self, attr, value):
         """Add an attribute to this element"""
         self.attributes[attr] = value
 
+    def setAttribute(self, attr, value):
+        """Set an attributes value - in fact does the same thing as add
+        attribute
+
+        """
+        self.attributes[attr] = value
+
     def setChars(self, chars):
         """Set the characters for this element"""
         self.chars = chars
-                
+
     def addChars(self, chars):
         """Add characters (textual value) to this element"""
         self.chars += chars
-    
+
     def addChild(self, child):
         """Add a child element to this element"""
         self.children.append(child)
         return self.children[len(self.children) - 1]
-        
+
     def getParent(self):
         """Get the parent of this element (Could be None)"""
         return self.parent
 
-    def setAttribute(self, attr, value):
-        """Set an attributes value - in fact does the same thing as add 
-        attribute
-        
-        """
-        self.attributes[attr] = value
-    
     def getChild(self, name):
-        """Returns a child element of name
-        
+        """Returns the first child element named 'name'
+
         Keywords:
         name -- The name of the child element to return"""
         for child in self.children:
             if child.name == name:
                 return child
-    
         return None
-                    
-    def get(self, element, attribute="", attrmatch=""):
-        """Return the data for either an attribute, or else an element"""
-        if (self.name == element):
+
+    def getChildren(self, name=None):
+        if name:
+            # return _all_ children named "name"
+            ret = []
+            for child in self.children:
+                if child.name == name:
+                    ret.append(child)
+            return ret
+        else:
+            return self.children
+
+    def get(self, elemName, attribute="", attrmatch=""):
+        """Return the text data for either an attribute or an xmlElement
+        """
+        if (self.name == elemName):
             if attribute != "":
-                if attrmatch != "": 
-                    if self.attributes[attribute] == attrmatch:
-                        return self.chars                
-                else:
-                    return self.attributes[attribute]
+                try:
+                    if attrmatch != "":
+                        if self.attributes[attribute] == attrmatch:
+                            return self.chars
+                    else:
+                        return self.attributes[attribute]
+                except AttributeError:
+                    return ""
             else:
                 return self.chars
-        
+
         for child in self.children:
-            if child.get(element, attribute, attrmatch) != "":
-                return child.get(element, attribute, attrmatch) 
-        
-        return ""         
+            ret = child.get(elemName, attribute, attrmatch)
+            if ret != "":
+                return ret
+
+        return ""
+
+
+
+class libpart():
+    """Class for a library part, aka 'libpart' in the xml netlist file.
+    (Components in eeschema are instantiated from library parts.)
+    This part class is implemented by wrapping an xmlElement with accessors.
+    This xmlElement instance is held in field 'element'.
+    """
+    def __init__(self, xml_element):
+        #
+        self.element = xml_element
+
+    #def __str__(self):
+        # simply print the xmlElement associated with this part
+        #return str(self.element)
+
+    def getLibName(self):
+        return self.element.get("libpart", "lib")
+
+    def getPartName(self):
+        return self.element.get("libpart", "part")
+
+    def getDescription(self):
+        return self.element.get("description")
+
+    def getField(self, name):
+        return self.element.get("field", "name", name)
+
+    def getFieldNames(self):
+        """Return a list of field names in play for this libpart.
+        """
+        fieldNames = []
+        fields = self.element.getChild('fields')
+        if fields:
+            for f in fields.getChildren():
+                fieldNames.append( f.get('field','name') )
+        return fieldNames
+
+    def getDatasheet(self):
+        return self.getField("Datasheet")
+
+    def getFootprint(self):
+        return self.getField("Footprint")
+
+    def getAliases(self):
+        """Return a list of aliases or None"""
+        aliases = self.element.getChild("aliases")
+        if aliases:
+            ret = []
+            children = aliases.getChildren()
+            # grab the text out of each child:
+            for child in children:
+                ret.append( child.get("alias") )
+            return ret
+        return None
+
+
+class comp():
+    """Class for a component, aka 'comp' in the xml netlist file.
+    This component class is implemented by wrapping an xmlElement instance
+    with accessors.  The xmlElement is held in field 'element'.
+    """
+
+    def __init__(self, xml_element):
+        self.element = xml_element
+        self.libpart = None
+
+        # Set to true when this component is included in a component group
+        self.grouped = False
+
+    def __eq__(self, other):
+        """Equlivalency operator, remember this can be easily overloaded"""
+        result = False
+        if self.getValue() == other.getValue():
+            if self.getLibName() == other.getLibName():
+                if self.getPartName() == other.getPartName():
+                    result = True
+        return result
+
+    def setLibPart(self, part):
+        self.libpart = part
+
+    def getLibPart(self):
+        return self.libpart
+
+    def getPartName(self):
+        return self.element.get("libsource", "part")
+
+    def getLibName(self):
+        return self.element.get("libsource", "lib")
+
+    def setValue(self, value):
+        """Set the value of this component"""
+        v = self.element.getChild("value")
+        if v:
+            v.setChars(value)
+
+    def getValue(self):
+        return self.element.get("value")
+
+    def getField(self, name, libraryToo=True):
+        """Return the value of a field named name. The component is first
+        checked for the field, and then the components library part is checked
+        for the field. If the field doesn't exist in either, an empty string is
+        returned
+
+        Keywords:
+        name -- The name of the field to return the value for
+        libraryToo --   look in the libpart's fields for the same name if not found
+                        in component itself
+        """
+
+        field = self.element.get("field", "name", name)
+        if field == "" and libraryToo:
+            field = self.libpart.getField(name)
+        return field
+
+    def getFieldNames(self):
+        """Return a list of field names in play for this component.  Mandatory
+        fields are not included, and they are: Value, Footprint, Datasheet, Ref.
+        The netlist format only includes fields with non-empty values.  So if a field
+        is empty, it will not be present in the returned list.
+        """
+        fieldNames = []
+        fields = self.element.getChild('fields')
+        if fields:
+            for f in fields.getChildren():
+                fieldNames.append( f.get('field','name') )
+        return fieldNames
+
+    def getRef(self):
+        return self.element.get("comp", "ref")
+
+    def getFootprint(self, libraryToo=True):
+        ret = self.element.get("footprint")
+        if ret =="" and libraryToo:
+            ret = self.libpart.getFootprint()
+        return ret
+
+    def getDatasheet(self, libraryToo=True):
+        ret = self.element.get("datasheet")
+        if ret == '' and libraryToo:
+            ret = self.libpart.getDatasheet()
+        return ret
+
+    def getTimestamp(self):
+        return self.element.get("tstamp")
+
+    def getDescription(self):
+        return self.libpart.getDescription()
 
 
 class netlist():
-    """ Kicad generic netlist class. Generally loaded from a kicad generic 
-    netlist file. Includes several helper functions to ease BOM creating 
+    """ Kicad generic netlist class. Generally loaded from a kicad generic
+    netlist file. Includes several helper functions to ease BOM creating
     scripts
-     
+
     """
     def __init__(self, fname=""):
         """Initialiser for the genericNetlist class
-        
+
         Keywords:
         fname -- The name of the generic netlist file to open (Optional)
-        
+
         """
         self.design = None
         self.components = []
         self.libparts = []
         self.libraries = []
         self.nets = []
-        
+
         # The entire tree is loaded into self.tree
         self.tree = []
 
         self._curr_element = None
-       
-        if fname != "":             
+
+        # component blacklist regexs, made from exluded_* above.
+        self.excluded_references = []
+        self.excluded_values = []
+        self.excluded_footprints = []
+
+        if fname != "":
             self.load(fname)
-        
+
     def addChars(self, content):
         """Add characters to the current element"""
-        self._curr_element.addChars(content) 
+        self._curr_element.addChars(content)
 
     def addElement(self, name):
         """Add a new kicad generic element to the list"""
         if self._curr_element == None:
-            self.tree = netlistElement(name)
-            self._curr_element = self.tree             
+            self.tree = xmlElement(name)
+            self._curr_element = self.tree
         else:
-            self._curr_element = self._curr_element.addChild( 
-                netlistElement(name, self._curr_element))
+            self._curr_element = self._curr_element.addChild(
+                xmlElement(name, self._curr_element))
 
-        # If this element is a component, add it to the components list 
+        # If this element is a component, add it to the components list
         if self._curr_element.name == "comp":
-            self.components.append(component(self._curr_element))
+            self.components.append(comp(self._curr_element))
 
         # Assign the design element
         if self._curr_element.name == "design":
@@ -295,17 +433,17 @@ class netlist():
 
         # If this element is a library part, add it to the parts list
         if self._curr_element.name == "libpart":
-            self.libparts.append(part(self._curr_element))
+            self.libparts.append(libpart(self._curr_element))
 
         # If this element is a net, add it to the nets list
         if self._curr_element.name == "net":
             self.nets.append(self._curr_element)
-        
+
         # If this element is a library, add it to the libraries list
         if self._curr_element.name == "library":
             self.libraries.append(self._curr_element)
-                
-        return self._curr_element        
+
+        return self._curr_element
 
     def endDocument(self):
         """Called when the netlist document has been fully parsed"""
@@ -314,8 +452,25 @@ class netlist():
         # duplicate library part information for every component
         for c in self.components:
             for p in self.libparts:
-                if p.getPart() == c.getPart() and p.getLib() == c.getLib():
-                    c.setPart(p)
+                if p.getLibName() == c.getLibName():
+                    if p.getPartName() == c.getPartName():
+                        c.setLibPart(p)
+                        break
+                    else:
+                        aliases = p.getAliases()
+                        if aliases and self.aliasMatch( c.getPartName(), aliases ):
+                            c.setLibPart(p)
+                            break;
+
+            if not c.getLibPart():
+                print( 'missing libpart for ref:', c.getRef(), c.getPartName(), c.getLibName() )
+
+
+    def aliasMatch(self, partName, aliasList):
+        for alias in aliasList:
+            if partName == alias:
+                return True
+        return False
 
     def endElement(self):
         """End the current element and switch to its parent"""
@@ -328,49 +483,196 @@ class netlist():
     def getSource(self):
         """Return the source string for the design"""
         return self.design.get("source")
-    
+
     def getTool(self):
         """Return the tool string which was used to create the netlist tree"""
         return self.design.get("tool")
 
-    def groupComponents(self):
+    def gatherComponentFieldUnion(self, components=None):
+        """Gather the complete 'set' of unique component fields, fields found in any component.
+        """
+        if not components:
+            components=self.components
+
+        s = set()
+        for c in components:
+            s.update( c.getFieldNames() )
+
+        # omit anything matching any regex in excluded_fields
+        ret = set()
+        for field in s:
+            exclude = False
+            for rex in excluded_fields:
+                if re.match( rex, field ):
+                    exclude = True
+                    break
+            if not exclude:
+                ret.add(field)
+
+        return ret       # this is a python 'set'
+
+    def gatherLibPartFieldUnion(self):
+        """Gather the complete 'set' of part fields, fields found in any part.
+        """
+        s = set()
+        for p in self.libparts:
+            s.update( p.getFieldNames() )
+
+        # omit anything matching any regex in excluded_fields
+        ret = set()
+        for field in s:
+            exclude = False
+            for rex in excluded_fields:
+                if re.match( rex, field ):
+                    exclude = True
+                    break
+            if not exclude:
+                ret.add(field)
+
+        return ret       # this is a python 'set'
+
+    def getInterestingComponents(self):
+        """Return a subset of all components, those that should show up in the BOM.
+        Omit those that should not, by consulting the blacklists:
+        excluded_values, excluded_refs, and excluded_footprints, which hold one
+        or more regular expressions.  If any of the the regular expressions match
+        the corresponding field's value in a component, then the component is exluded.
+        """
+
+        # pre-compile all the regex expressions:
+        del self.excluded_references[:]
+        del self.excluded_values[:]
+        del self.excluded_footprints[:]
+
+        for rex in excluded_references:
+            self.excluded_references.append( re.compile( rex ) )
+
+        for rex in excluded_values:
+            self.excluded_values.append( re.compile( rex ) )
+
+        for rex in excluded_footprints:
+            self.excluded_footprints.append( re.compile( rex ) )
+
+        # the subset of components to return, considered as "interesting".
+        ret = []
+
+        # run each component thru a series of tests, if it passes all, then add it
+        # to the interesting list 'ret'.
+        for c in self.components:
+            exclude = False
+            if not exclude:
+                for refs in self.excluded_references:
+                    if refs.match(c.getRef()):
+                        exclude = True
+                        break;
+            if not exclude:
+                for vals in self.excluded_values:
+                    if vals.match(c.getValue()):
+                        exclude = True
+                        break;
+            if not exclude:
+                for mods in self.excluded_footprints:
+                    if mods.match(c.getFootprint()):
+                        exclude = True
+                        break;
+
+            if not exclude:
+                # This is a fairly personal way to flag DNS (Do Not Stuff).  NU for
+                # me means Normally Uninstalled.  You can 'or in' another expression here.
+                if c.getField( "Installed" ) == 'NU':
+                    exclude = True
+
+            if not exclude:
+                ret.append(c)
+
+        # Sort first by ref as this makes for easier to read BOM's
+        ret.sort(key=lambda g: g.getRef())
+
+        return ret
+
+
+    def groupComponents(self, components = None):
         """Return a list of component lists. Components are grouped together
-        when the value, library and part identifiers match
-        
+        when the value, library and part identifiers match.
+
+        Keywords:
+        components -- is a list of components, typically an interesting subset
+        of all components, or None.  If None, then all components are looked at.
         """
+        if not components:
+            components = self.components
+
         groups = []
 
-        # Make sure to start off will all components ungrouped to begin with        
-        for c in self.components:
+        # Make sure to start off will all components ungrouped to begin with
+        for c in components:
             c.grouped = False
 
         # Group components based on the value, library and part identifiers
-        for c in self.components:
+        for c in components:
             if c.grouped == False:
                 c.grouped = True
                 newgroup = []
                 newgroup.append(c)
-                
+
                 # Check every other ungrouped component against this component
-                # and add to the group as necessary                
-                for ci in self.components:                
+                # and add to the group as necessary
+                for ci in components:
                     if ci.grouped == False and ci == c:
                         newgroup.append(ci)
                         ci.grouped = True
-                
-                # Add the new component group to the groups list                
+
+                # Add the new component group to the groups list
                 groups.append(newgroup)
-        
+
         # Each group is a list of components, we need to sort each list first
         # to get them in order as this makes for easier to read BOM's
         for g in groups:
             g = sorted(g, key=lambda g: g.getRef())
-            
+
         # Finally, sort the groups to order the references alphabetically
         groups = sorted(groups, key=lambda group: group[0].getRef())
-                
+
         return groups
-    
+
+    def getGroupField(self, group, field):
+        """Return the whatever is known about the given field by consulting each
+        component in the group.  If any of them know something about the property/field,
+        then return that first non-blank value.
+        """
+        for c in group:
+            ret = c.getField(field, False)
+            if ret != '':
+                return ret
+        return group[0].getLibPart().getField(field)
+
+    def getGroupFootprint(self, group):
+        """Return the whatever is known about the Footprint by consulting each
+        component in the group.  If any of them know something about the Footprint,
+        then return that first non-blank value.
+        """
+        for c in group:
+            ret = c.getFootprint()
+            if ret != "":
+                return ret
+        return group[0].getLibPart().getFootprint()
+
+    def getGroupDatasheet(self, group):
+        """Return the whatever is known about the Datasheet by consulting each
+        component in the group.  If any of them know something about the Datasheet,
+        then return that first non-blank value.
+        """
+        for c in group:
+            ret = c.getDatasheet()
+            if ret != "":
+                return ret
+
+        if len(group) > 0:
+            return group[0].getLibPart().getDatasheet()
+        else:
+            print("NULL!")
+        return ''
+
     def formatXML(self):
         """Return the whole netlist formatted in XML"""
         return self.tree.formatXML()
@@ -378,73 +680,48 @@ class netlist():
     def formatHTML(self):
         """Return the whole netlist formatted in HTML"""
         return self.tree.formatHTML()
-                                     
+
     def load(self, fname):
         """Load a kicad generic netlist
-        
+
         Keywords:
-        fname -- The name of the generic netlist file to open 
-        
+        fname -- The name of the generic netlist file to open
+
         """
         try:
             self._reader = sax.make_parser()
             self._reader.setContentHandler(_gNetReader(self))
             self._reader.parse(fname)
         except IOError as e:
-            print >> sys.stderr, __file__, ":", e
-            sys.exit(-1)    
-
-
-class part():
-    """Class for a library part"""
-    def __init__(self, part):
-        # The part is a reference to a libpart generic netlist element
-        self.element = part
-    
-    def __str__(self):
-        # simply print the generic netlist element associated with this part
-        return str(self.element)
-        
-    def getDatasheet(self):
-        return self.element.get("docs")
+            print( __file__, ":", e, file=sys.stderr )
+            sys.exit(-1)
 
-    def getLib(self):
-        return self.element.get("libpart", "lib")
-    
-    def getPart(self):
-        return self.element.get("libpart", "part")
-
-    def getDescription(self):
-        return self.element.get("description")
-                
-    def getField(self, name):
-        return self.element.get("field", "name", name)
 
 
 class _gNetReader(sax.handler.ContentHandler):
     """SAX kicad generic netlist content handler - passes most of the work back
-    to the gNetlist class which builds a complete tree in RAM for the design
-    
+    to the 'netlist' class which builds a complete tree in RAM for the design
+
     """
     def __init__(self, aParent):
         self.parent = aParent
-    
+
     def startElement(self, name, attrs):
         """Start of a new XML element event"""
         element = self.parent.addElement(name)
-        
+
         for name in attrs.getNames():
             element.addAttribute(name, attrs.getValue(name))
-        
+
     def endElement(self, name):
         self.parent.endElement()
-    
+
     def characters(self, content):
         # Ignore erroneous white space - ignoreableWhitespace does not get rid
         # of the need for this!
         if not content.isspace():
             self.parent.addChars(content)
-            
+
     def endDocument(self):
         """End of the XML document event"""
         self.parent.endDocument()

scripts/bom-in-python/round_robin.py

@@ -4,20 +4,24 @@
 # Example: Round robin, XML to XML conversion
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import sys
+import pdb
+
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # the command line option. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write to, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print( __file__, ":", e, file=sys.stderr)
     f = stdout
 
-print >> f, net.formatXML()
+print(net.formatXML(), file=f)

scripts/bom-in-python/round_value_robin.py

@@ -4,8 +4,10 @@
 # Example: Round value robin, XML to XML conversion with partial value monging
 #
 
+from __future__ import print_function
+
 # Import the KiCad python helper module and the csv formatter
-import ky_generic_netlist_reader
+import kicad_netlist_reader
 import sys
 
 def checkvalue(self):
@@ -51,21 +53,21 @@ def checkvalue(self):
 
 # Give components a new method for checking the values (this could easily be a
 # Company Part Number generator method instead)
-ky_generic_netlist_reader.component.checkvalue = checkvalue
+kicad_netlist_reader.comp.checkvalue = checkvalue
 
 # Generate an instance of a generic netlist, and load the netlist tree from
 # the command line option. If the file doesn't exist, execution will stop
-net = ky_generic_netlist_reader.netlist(sys.argv[1])
+net = kicad_netlist_reader.netlist(sys.argv[1])
 
 # Open a file to write to, if the file cannot be opened output to stdout
 # instead
 try:
     f = open(sys.argv[2], 'w')
 except IOError:
-    print >> sys.stderr, __file__, ":", e
+    print(__file__, ":", e, file=sys.stderr)
     f = stdout
 
 for c in net.components:
     c.checkvalue()
 
-print >> f, net.formatXML()
+print(net.formatXML(), file=f)