Added global compilation option/pragma support to parser and command line

Cython/Compiler/CmdLine.py

@@ -37,6 +37,7 @@ Options:
   -a, --annotate                 Produce an colorized version of the source.
   --convert-range                Convert for loops using range() function to for...from loops. 
   --cplus                        Output a c++ rather than c file.
+  -O, --option <name>=<value>[,<name=value,...] Overrides an optimization/code generation option
 """
 #The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
@@ -110,6 +111,12 @@ def parse_command_line(args):
                 Options.annotate = True
             elif option == "--convert-range":
                 Options.convert_range = True
+            elif option in ("-O", "--option"):
+                try:
+                    options.pragma_overrides = Options.parse_option_list(pop_arg())
+                except ValueError, e:
+                    sys.stderr.write("Error in option string: %s\n" % e.message)
+                    sys.exit(1)
             else:
                 bad_usage()
         else:

Cython/Compiler/ExprNodes.py

@@ -1525,6 +1525,7 @@ class IndexNode(ExprNode):
         self.generate_subexpr_disposal_code(code)
 
     def buffer_access_code(self, code):
+        print self.options
         # Assign indices to temps
         index_temps = [code.funcstate.allocate_temp(i.type) for i in self.indices]
         for temp, index in zip(index_temps, self.indices):

Cython/Compiler/Lexicon.py

@@ -66,6 +66,7 @@ def make_lexicon():
     escapeseq = Str("\\") + (two_oct | three_oct | two_hex |
                              Str('u') + four_hex | Str('x') + two_hex | AnyChar)
     
+
     deco = Str("@")
     bra = Any("([{")
     ket = Any(")]}")
@@ -74,9 +75,12 @@ def make_lexicon():
                     "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", 
                     "<<=", ">>=", "**=", "//=")
     spaces = Rep1(Any(" \t\f"))
-    comment = Str("#") + Rep(AnyBut("\n"))
     escaped_newline = Str("\\\n")
     lineterm = Eol + Opt(Str("\n"))
+
+    comment_start = Str("#")
+    comment = comment_start + Rep(AnyBut("\n"))    
+    option_comment = comment_start + Str("cython:") + Rep(AnyBut("\n"))
     
     return Lexicon([
         (name, 'IDENT'),
@@ -93,11 +97,13 @@ def make_lexicon():
         #(stringlit, 'STRING'),
         (beginstring, Method('begin_string_action')),
         
+        (option_comment, 'option_comment'),
         (comment, IGNORE),
         (spaces, IGNORE),
         (escaped_newline, IGNORE),
         
         State('INDENT', [
+            (option_comment + lineterm, 'option_comment'),
             (Opt(spaces) + Opt(comment) + lineterm, IGNORE),
             (indentation, Method('indentation_action')),
             (Eof, Method('eof_action'))

Cython/Compiler/Main.py

@@ -58,12 +58,13 @@ class Context:
     #  include_directories   [string]
     #  future_directives     [object]
     
-    def __init__(self, include_directories):
+    def __init__(self, include_directories, pragma_overrides):
         #self.modules = {"__builtin__" : BuiltinScope()}
         import Builtin
         self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.include_directories = include_directories
         self.future_directives = set()
+        self.pragma_overrides = pragma_overrides
 
         self.pxds = {} # full name -> node tree
 
@@ -76,6 +77,7 @@ class Context:
         from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
         from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
         from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
+        from ParseTreeTransforms import ResolveOptions
         from Optimize import FlattenInListTransform, SwitchTransform, OptimizeRefcounting
         from Buffer import IntroduceBufferAuxiliaryVars
         from ModuleNode import check_c_classes
@@ -91,6 +93,7 @@ class Context:
             NormalizeTree(self),
             PostParse(self),
             _specific_post_parse,
+            ResolveOptions(self, self.pragma_overrides),
             FlattenInListTransform(),
             WithTransform(self),
             DecoratorTransform(self),
@@ -481,7 +484,7 @@ def create_default_resultobj(compilation_source, options):
 
 def run_pipeline(source, options, full_module_name = None):
     # Set up context
-    context = Context(options.include_path)
+    context = Context(options.include_path, options.pragma_overrides)
 
     # Set up source object
     cwd = os.getcwd()
@@ -531,6 +534,7 @@ class CompilationOptions:
                                 defaults to true when recursive is true.
     verbose           boolean   Always print source names being compiled
     quiet             boolean   Don't print source names in recursive mode
+    pragma_overrides  dict      Overrides for pragma options (see Options.py)
     
     Following options are experimental and only used on MacOSX:
     
@@ -723,7 +727,9 @@ default_options = dict(
     recursive = 0,
     timestamps = None,
     verbose = 0,
-    quiet = 0)
+    quiet = 0,
+    pragma_overrides = {}
+)
 if sys.platform == "mac":
     from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError
     default_options['use_listing_file'] = 1

Cython/Compiler/Options.py

 #
-#  Pyrex - Compilation-wide options
+#  Cython - Compilation-wide options and pragma declarations
 #
 
 cache_builtins = 1  #  Perform lookups on builtin names only once
@@ -52,3 +52,58 @@ optimize_simple_methods = 1
 
 # Append the c file and line number to the traceback for exceptions. 
 c_line_in_traceback = 1
+
+
+# Declare pragmas
+option_types = {
+    'boundscheck' : bool
+}
+
+option_defaults = {
+    'boundscheck' : True
+}
+
+def parse_option_list(s):
+    """
+    Parses a comma-seperated list of pragma options. Whitespace
+    is not considered.
+
+    >>> parse_option_list('      ')
+    {}
+    >>> (parse_option_list('boundscheck=True') ==
+    ... {'boundscheck': True})
+    True
+    >>> parse_option_list('  asdf')
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected "=" in option "asdf"
+    >>> parse_option_list('boundscheck=hey')
+    Traceback (most recent call last):
+       ...
+    ValueError: Must pass a boolean value for option "boundscheck"
+    >>> parse_option_list('unknown=True')
+    Traceback (most recent call last):
+       ...
+    ValueError: Unknown option: "unknown"
+    """
+    result = {}
+    for item in s.split(','):
+        item = item.strip()
+        if not item: continue
+        if not '=' in item: raise ValueError('Expected "=" in option "%s"' % item)
+        name, value = item.strip().split('=')
+        try:
+            type = option_types[name]
+        except KeyError:
+            raise ValueError('Unknown option: "%s"' % name)
+        if type is bool:
+            value = value.lower()
+            if value in ('true', 'yes'):
+                value = True
+            elif value in ('false', 'no'):
+                value = False
+            else: raise ValueError('Must pass a boolean value for option "%s"' % name)
+            result[name] = value
+        else:
+            assert False
+    return result

Cython/Compiler/ParseTreeTransforms.py

@@ -6,6 +6,7 @@ from Cython.Compiler.TreeFragment import TreeFragment
 from Cython.Utils import EncodedString
 from Cython.Compiler.Errors import CompileError
 from sets import Set as set
+import copy
 
 class NormalizeTree(CythonTransform):
     """
@@ -253,6 +254,37 @@ class PxdPostParse(CythonTransform):
         else:
             return node
 
+class ResolveOptions(CythonTransform):
+    """
+    After parsing, options can be stored in a number of places:
+    - #cython-comments at the top of the file (stored in ModuleNode)
+    - Command-line arguments overriding these
+    - @cython.optionname decorators
+    - with cython.optionname: statements
+
+    This transform is responsible for annotating each node with an
+    "options" attribute linking it to a dict containing the exact
+    options that are in effect for that node. Any corresponding decorators
+    or with statements are removed in the process.
+    """
+
+    def __init__(self, context, compilation_option_overrides):
+        super(ResolveOptions, self).__init__(context)
+        self.compilation_option_overrides = compilation_option_overrides
+
+    def visit_ModuleNode(self, node):
+        options = copy.copy(Options.option_defaults)
+        options.update(node.option_comments)
+        options.update(self.compilation_option_overrides)
+        self.options = options
+        node.options = options
+        self.visitchildren(node)
+        return node
+
+    def visit_Node(self, node):
+        node.options = self.options
+        self.visitchildren(node)
+        return node
 
 class WithTransform(CythonTransform):
 

Cython/Compiler/Parsing.py

@@ -11,6 +11,7 @@ from ModuleNode import ModuleNode
 from Errors import error, warning, InternalError
 from Cython import Utils
 import Future
+import Options
 
 class Ctx(object):
     #  Parsing context
@@ -506,6 +507,8 @@ def p_atom(s):
     elif sy == 'NULL':
         s.next()
         return ExprNodes.NullNode(pos)
+    elif sy == 'option_comment':
+        s.error("#cython option comments only allowed at beginning of file")
     else:
         s.error("Expected an identifier or literal")
 
@@ -2309,6 +2312,17 @@ def p_code(s, level=None):
             repr(s.sy), repr(s.systring)))
     return body
 
+def p_option_comments(s):
+    result = {}
+    while s.sy == 'option_comment':
+        opts = s.systring[len("#cython:"):]
+        try:
+            result.update(Options.parse_option_list(opts))
+        except ValueError, e:
+            s.error(e.message, fatal=False)
+        s.next()
+    return result
+
 def p_module(s, pxd, full_module_name):
     s.add_type_name("object")
     s.add_type_name("Py_buffer")
@@ -2318,11 +2332,15 @@ def p_module(s, pxd, full_module_name):
         level = 'module_pxd'
     else:
         level = 'module'
+
+    option_comments = p_option_comments(s)
     body = p_statement_list(s, Ctx(level = level), first_statement = 1)
     if s.sy != 'EOF':
         s.error("Syntax error in statement [%s,%s]" % (
             repr(s.sy), repr(s.systring)))
-    return ModuleNode(pos, doc = doc, body = body, full_module_name = full_module_name)
+    return ModuleNode(pos, doc = doc, body = body,
+                      full_module_name = full_module_name,
+                      option_comments = option_comments)
 
 #----------------------------------------------
 #

Cython/Compiler/Scanning.py

@@ -429,12 +429,13 @@ class PyrexScanner(Scanner):
     def looking_at_type_name(self):
         return self.sy == 'IDENT' and self.systring in self.type_names
     
-    def error(self, message, pos = None):
+    def error(self, message, pos = None, fatal = True):
         if pos is None:
             pos = self.position()
         if self.sy == 'INDENT':
-            error(pos, "Possible inconsistent indentation")
-        raise error(pos, message)
+            err = error(pos, "Possible inconsistent indentation")
+        err = error(pos, message)
+        if fatal: raise err
         
     def expect(self, what, message = None):
         if self.sy == what:

Cython/Compiler/TreeFragment.py

@@ -20,7 +20,7 @@ Support for parsing strings into code trees.
 
 class StringParseContext(Main.Context):
     def __init__(self, include_directories, name):
-        Main.Context.__init__(self, include_directories)
+        Main.Context.__init__(self, include_directories, {})
         self.module_name = name
         
     def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):

tests/errors/e_options.pyx

+
+#cython: nonexistant
+#cython: some=9
+
+# The one below should NOT raise an error
+
+#cython: boundscheck=True
+
+# However this one should
+#cython: boundscheck=sadf
+
+print 3
+
+#cython: boundscheck=True
+
+_ERRORS = u"""
+2:0: Expected "=" in option "nonexistant"
+3:0: Unknown option: "some"
+10:0: Must pass a boolean value for option "boundscheck"
+14:0: #cython option comments only allowed at beginning of file
+"""
+

tests/run/options.pyx

+#cython: boundscheck=False
+
+def f(object[int, 2] buf):
+    print buf[3, 2]