Merge pull request #76 from pombredanne/header_expressions

Add support to ignore sections conditionally based on a Python expression.

src/zc/buildout/buildout.py

@@ -1400,6 +1400,85 @@ def _save_options(section, options, f):
     for option, value in items:
         _save_option(option, value, f)
 
+def _default_globals():
+    """Return a mapping of default and precomputed expressions.
+    These default expressions are convenience defaults available when eveluating
+    section headers expressions.
+    NB: this is wrapped in a function so that the computing of these expressions
+    is lazy and done only if needed (ie if there is at least one section with 
+    an expression) because the computing of some of these expressions can be 
+    expensive. 
+    """
+    # partially derived or inspired from its.py
+    # Copyright (c) 2012, Kenneth Reitz All rights reserved.
+    # Redistribution and use in source and binary forms, with or without modification,
+    # are permitted provided that the following conditions are met:
+    # Redistributions of source code must retain the above copyright notice, this list
+    # of conditions and the following disclaimer. Redistributions in binary form must
+    # reproduce the above copyright notice, this list of conditions and the following
+    # disclaimer in the documentation and/or other materials provided with the
+    # distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+    # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+    # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+    # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+    # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+    # OF SUCH DAMAGE.
+
+    # default available modules, explicitly re-imported locally here on purpose
+    import sys
+    import os
+    import platform
+    import re
+
+    globals_defs = {'sys': sys, 'os': os, 'platform': platform, 're': re,}
+
+    # major python major_python_versions as python2 and python3
+    major_python_versions = platform.python_version_tuple()
+    globals_defs.update({'python2': major_python_versions[0] == '2',
+                         'python3': major_python_versions[0] == '3'})
+
+    # minor python major_python_versions as python24, python25 ... python36
+    minor_python_versions = ('24', '25', '26', '27',
+                             '30', '31', '32', '33', '34', '35', '36')
+    for v in minor_python_versions:
+        globals_defs['python' + v] = ''.join(major_python_versions[:2]) == v
+
+    # interpreter type
+    sys_version = sys.version.lower()
+    pypy = 'pypy' in sys_version
+    jython = 'java' in sys_version
+    ironpython ='iron' in sys_version
+    # assume CPython, if nothing else.
+    cpython = not any((pypy, jython, ironpython,))
+    globals_defs.update({'cpython': cpython,
+                         'pypy': pypy,
+                         'jython': jython,
+                         'ironpython': ironpython})
+
+    # operating system
+    sys_platform = str(sys.platform).lower()
+    globals_defs.update({'linux': 'linux' in sys_platform,
+                         'windows': 'win32' in sys_platform,
+                         'cygwin': 'cygwin' in sys_platform,
+                         'solaris': 'sunos' in sys_platform,
+                         'macosx': 'darwin' in sys_platform,
+                         'posix': 'posix' in os.name.lower()})
+
+    #bits and endianness
+    import struct
+    void_ptr_size = struct.calcsize('P') * 8
+    globals_defs.update({'bits32': void_ptr_size == 32,
+                         'bits64': void_ptr_size == 64,
+                         'little_endian': sys.byteorder == 'little',
+                         'big_endian': sys.byteorder == 'big'})
+
+    return globals_defs
+
 def _open(base, filename, seen, dl_options, override, downloaded):
     """Open a configuration file and return the result as a dictionary,
 
@@ -1441,7 +1520,7 @@ def _open(base, filename, seen, dl_options, override, downloaded):
     root_config_file = not seen
     seen.append(filename)
 
-    result = zc.buildout.configparser.parse(fp, filename)
+    result = zc.buildout.configparser.parse(fp, filename, _default_globals)
     fp.close()
     if is_temp:
         os.remove(path)

src/zc/buildout/configparser.py

@@ -19,6 +19,9 @@
 
 import re
 import textwrap
+import logging
+
+logger = logging.getLogger('zc.buildout')
 
 class Error(Exception):
     """Base class for ConfigParser exceptions."""
@@ -71,25 +74,68 @@ class MissingSectionHeaderError(ParsingError):
         self.lineno = lineno
         self.line = line
 
+# This regex captures either sections headers with optional trailing comment
+# separated by a semicolon or a hash.  Section headers can have an optional
+# expression. Expressions and comments can contain brackets but no verbatim '#'
+# and ';' : these need to be escaped.
+# A title line with an expression has the general form:
+#  [section_name: some Python expression] #; some comment
+# This regex leverages the fact that the following is a valid Python expression:
+#  [some Python expression] # some comment
+# and that section headers are also delimited by [brackets] taht are also [list]
+# delimiters.
+# So instead of doing complex parsing to balance brackets in an expression, we
+# capture just enough from a header line to collect then remove the section_name
+# and colon expression separator keeping only a list-enclosed expression and
+# optional comments. The parsing and validation of this Python expression can be
+# entirely delegated to Python's eval. The result of the evaluated expression is
+# the always returned wrapped in a list with a single item that contains the
+# original expression
+
 section_header  = re.compile(
-    r'\[\s*(?P<header>[^\s[\]:{}]+)\s*]\s*([#;].*)?$').match
+    r'(?P<head>\[)'
+    r'\s*'
+    r'(?P<name>[^\s#[\]:;{}]+)'
+    r'\s*'
+    r'(:(?P<expression>[^#;]*))?'
+    r'\s*'
+    r'(?P<tail>]'
+    r'\s*'
+    r'([#;].*)?$)'
+    ).match
+
 option_start = re.compile(
     r'(?P<name>[^\s{}[\]=:]+\s*[-+]?)'
     r'='
     r'(?P<value>.*)$').match
+
 leading_blank_lines = re.compile(r"^(\s*\n)+")
 
-def parse(fp, fpname):
+def parse(fp, fpname, exp_globals=None):
     """Parse a sectioned setup file.
 
-    The sections in setup file contains a title line at the top,
+    The sections in setup files contain a title line at the top,
     indicated by a name in square brackets (`[]'), plus key/value
     options lines, indicated by `name: value' format lines.
     Continuations are represented by an embedded newline then
     leading whitespace.  Blank lines, lines beginning with a '#',
     and just about everything else are ignored.
+
+    The title line is in the form [name] followed by an optional trailing 
+    comment separated by a semicolon `;' or a hash `#' character. 
+    
+    Optionally the title line can have the form `[name:expression]' where 
+    expression is an arbitrary Python expression. Sections with an expression 
+    that evaluates to False are ignored. Semicolon `;' an hash `#' characters
+    mustr be string-escaped in expression literals.
+
+    exp_globals is a callable returning a mapping of defaults used as globals 
+    during the evaluation of a section conditional expression.
     """
     sections = {}
+    # the current section condition, possibly updated from a section expression
+    section_condition = True
+    context = None
     cursect = None                            # None, or a dictionary
     blockmode = None
     optname = None
@@ -106,6 +152,9 @@ def parse(fp, fpname):
             continue # comment
 
         if line[0].isspace() and cursect is not None and optname:
+            if not section_condition:
+                #skip section based on its expression condition
+                continue
             # continuation line
             if blockmode:
                 line = line.rstrip()
@@ -115,10 +164,32 @@ def parse(fp, fpname):
                     continue
             cursect[optname] = "%s\n%s" % (cursect[optname], line)
         else:
-            mo = section_header(line)
-            if mo:
-                # section header
-                sectname = mo.group('header')
+            header = section_header(line)
+            if header:
+                # reset to True when starting a new section
+                section_condition = True 
+                sectname = header.group('name')
+
+                head = header.group('head') # the starting [
+                expression = header.group('expression')
+                tail = header.group('tail') # closing ]and comment
+                if expression:
+                    # normalize tail comments to Python style
+                    tail = tail.replace(';', '#') if tail else ''
+                    # un-escape literal # and ; . Do not use a string-escape decode
+                    expr = expression.replace(r'\x23','#').replace(r'x3b', ';') 
+                    # rebuild a valid Python expression wrapped in a list
+                    expr = head + expr + tail
+                    # lazily populate context only expression
+                    if not context:
+                        context = exp_globals() if exp_globals else {}
+                    # evaluated expression is in list: get first element
+                    section_condition = eval(expr, context)[0]
+                    # finally, ignore section when an expression evaluates to false
+                    if not section_condition:
+                        logger.debug('Ignoring section %(sectname)r with [expression]: %(expression)r' % locals())
+                        continue
+
                 if sectname in sections:
                     cursect = sections[sectname]
                 else:
@@ -133,6 +204,9 @@ def parse(fp, fpname):
             else:
                 mo = option_start(line)
                 if mo:
+                    if not section_condition:
+                        # filter out options of conditionally ignored section
+                        continue
                     # option start line
                     optname, optval = mo.group('name', 'value')
                     optname = optname.rstrip()

src/zc/buildout/configparser.test

@@ -89,3 +89,260 @@ otherwise empty section) is blank.  For example:"
                'on_update': 'true',
                'recipe': 'collective.recipe.cmd'},
      'versions': {}}
+
+
+Sections headers can contain an optional arbitrary Python expression. 
+When the expression evaluates to false the whole section is skipped.
+Several sections can have the same name with different expressions, enabling 
+conditional exclusion of sections::
+
+  [s1: 2 + 2 == 4] # this expression is true [therefore "this section" _will_ be NOT skipped
+  a = 1
+
+  [   s2 : 2 + 2 == 5  ]         # comment: this expression is false, so this section will be ignored]
+  long = a
+
+  [   s2 : 41 + 1 == 42  ]  # a comment: this expression is [true], so this section will be kept
+  long = b
+
+  [s3:2 in map(lambda i:i*2, [i for i in range(10)])] ;# Complex expressions are [possible!];, though they should not be (abused:) 
+  # this section will not be skipped
+  long = c
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'s1': {'a': '1'}, 's2': {'long': 'b'}, 's3': {'long': 'c'}}
+
+
+Title line optional trailing comments are separated by a hash '#' or semicolon
+';' character.  The expression is an arbitrary expression with one restriction: 
+it cannot contain a literal hash '#' or semicolon ';' character: these need to be 
+string-escaped.
+The comment can contain arbitrary characters, including brackets that are also 
+used to mark the end of a section header and may be ambiguous to recognize in 
+some cases. For example, valid sections lines include::
+
+  [ a ]
+  a=1
+
+  [ b ]  # []
+  b=1
+
+  [ c : True ]  # ]
+  c =1
+
+  [ d :  True]  # []
+  d=1
+
+  [ e ]  # []
+  e = 1
+
+  [ f ]  # ]
+  f = 1
+
+  [g:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] # Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
+  g = 1
+
+  [ h : True ]  ; ]
+  h =1
+
+  [ i :  True]  ; []
+  i=1
+
+  [j:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] ; Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
+  j = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'a': {'a': '1'},
+     'b': {'b': '1'},
+     'c': {'c': '1'},
+     'd': {'d': '1'},
+     'e': {'e': '1'},
+     'f': {'f': '1'},
+     'g': {'g': '1'},
+     'h': {'h': '1'},
+     'i': {'i': '1'},
+     'j': {'j': '1'}}
+
+
+
+A title line optional trailing comment be separated by a hash or semicolon 
+character. The following are valid semicolon-separated comments::
+
+  [ a ]  ;semicolon comment are supported for lines without expressions ]
+  a = 1
+
+  [ b ]  ; []
+  b = 1
+
+  [ c ]  ; ]
+  c = 1
+
+  [ d ]  ; [
+  d = 1
+
+  [ e: True ]  ;semicolon comments are supported for lines with expressions ]
+  e = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'a': {'a': '1'},
+     'b': {'b': '1'},
+     'c': {'c': '1'},
+     'd': {'d': '1'},
+     'e': {'e': '1'}}
+
+
+
+The following sections with hash comment separators are valid too::
+
+  [ a ]  #hash comment ] are supported for lines without expressions ]
+  a = 1
+
+  [ b ]  # []
+  b = 1
+
+  [ c ]  # ]
+  c = 1
+
+  [ d ]  # [
+  d = 1
+
+  [ e: True ]  #hash comments] are supported for lines with expressions ]
+  e = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'a': {'a': '1'},
+     'b': {'b': '1'},
+     'c': {'c': '1'},
+     'd': {'d': '1'},
+     'e': {'e': '1'}}
+
+
+However, explicit semicolon and hash characters are invalid in expressions and 
+must be escaped or this triggers an error. In the rare case where a hash '#' or
+semicolon ';' would be needed in an expression literal, you can use the 
+string-escaped representation of these characters:  use '\x23' for hash '#' and 
+'\x3b' for semicolon ';' to avoid evaluation errors.
+These expressions are valid and use escaped hash and semicolons in literals::
+
+  [a:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] # Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
+  a = 1
+
+  [b:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] ; Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
+  b = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'a': {'a': '1'}, 'b': {'b': '1'}}
+
+
+And using unescaped semicolon and hash characters in expressions triggers an error::
+
+  [a:'#' in '#;'] # this is not a supported expression
+  a = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import zc.buildout.configparser
+    >>> try: zc.buildout.configparser.parse(StringIO.StringIO(text), 'test')
+    ... except zc.buildout.configparser.MissingSectionHeaderError: pass # success
+
+
+One of the typical usage of expression is to have buildout parts that are 
+operating system or platform-specific.  The configparser.parse function has an 
+optional exp_globals argument.  This is a callable returning a mapping of 
+objects made available to the evaluation context of the expression. Here we add 
+the platform and sys modules to the evaluation context, so we can access 
+platform and sys modules functions and objects in our expressions ::
+
+  [s1: platform.python_version_tuple()[0] in ('2', '3',)] # this expression is true, the major versions of python are either 2 or 3
+  a = 1
+
+  [s2:sys.version[0] == '0'] # comment: this expression "is false",  there no major version 0 of Python so this section will be ignored
+  long = a
+
+  [s2:len(platform.uname()) > 0]  # a comment: this expression is likely always true, so this section will be kept
+  long = b
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> import platform, sys
+    >>> globs = lambda: {'platform': platform, 'sys': sys}
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test', exp_globals=globs))
+    {'s1': {'a': '1'}, 's2': {'long': 'b'}}
+
+
+Some limited (but hopefully sane and sufficient) default modules and 
+pre-computed common expressions available to an expression when the parser in 
+called by buildout::
+
+
+  #imported modules
+  [s1: sys and re and os and platform] # this expression is true: these modules are available
+  a = 1
+
+  # major and minor python versions, yes even python 3.5 and 3.6 are there , prospectively
+  # comment: this expression "is true" and not that long expression cannot span several lines 
+  [s2: any([python2, python3, python24 , python25 , python26 , python27 , python30 , python31 , python32 , python33 , python34 , python35 , python36]) ] 
+  b = 1
+
+  # common python interpreter types
+  [s3:cpython or pypy or jython or ironpython]  # a comment: this expression is likely always true, so this section will be kept
+  c = 1 
+
+  # common operating systems
+  [s4:linux or windows or cygwin or macosx or solaris or posix or True]  
+  d = 1
+
+  # common bitness and endianness
+  [s5:bits32 or bits64 or little_endian or big_endian]  
+  e = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> import zc.buildout.buildout
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test', zc.buildout.buildout._default_globals))
+    {'s1': {'a': '1'},
+     's2': {'b': '1'},
+     's3': {'c': '1'},
+     's4': {'d': '1'},
+     's5': {'e': '1'}}
+