Commit 6318423f authored by Brock Mendel's avatar Brock Mendel

REF: Move CompilationOptions and default_options to Options

update imports
remove Options.create_context in favor of Context.from_options
parent e80bbb3e
...@@ -42,7 +42,8 @@ except: ...@@ -42,7 +42,8 @@ except:
from .. import Utils from .. import Utils
from ..Utils import (cached_function, cached_method, path_exists, from ..Utils import (cached_function, cached_method, path_exists,
safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix)
from ..Compiler.Main import Context, CompilationOptions, default_options from ..Compiler.Main import Context
from ..Compiler.Options import CompilationOptions, default_options
join_path = cached_function(os.path.join) join_path = cached_function(os.path.join)
copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) copy_once_if_newer = cached_function(copy_file_to_dir_if_newer)
...@@ -955,7 +956,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, ...@@ -955,7 +956,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
c_options = CompilationOptions(**options) c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options); cpp_options.cplus = True cpp_options = CompilationOptions(**options); cpp_options.cplus = True
ctx = c_options.create_context() ctx = Context.from_options(c_options)
options = c_options options = c_options
module_list, module_metadata = create_extension_list( module_list, module_metadata = create_extension_list(
module_list, module_list,
......
...@@ -8,7 +8,8 @@ from distutils.core import Distribution, Extension ...@@ -8,7 +8,8 @@ from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext from distutils.command.build_ext import build_ext
import Cython import Cython
from ..Compiler.Main import Context, CompilationOptions, default_options from ..Compiler.Main import Context
from ..Compiler.Options import CompilationOptions, default_options
from ..Compiler.ParseTreeTransforms import (CythonTransform, from ..Compiler.ParseTreeTransforms import (CythonTransform,
SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform) SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform)
......
...@@ -65,8 +65,6 @@ def bad_usage(): ...@@ -65,8 +65,6 @@ def bad_usage():
def parse_command_line(args): def parse_command_line(args):
from .Main import CompilationOptions, default_options
pending_arg = [] pending_arg = []
def pop_arg(): def pop_arg():
...@@ -94,7 +92,7 @@ def parse_command_line(args): ...@@ -94,7 +92,7 @@ def parse_command_line(args):
else: else:
return pop_arg() return pop_arg()
options = CompilationOptions(default_options) options = Options.CompilationOptions(Options.default_options)
sources = [] sources = []
while args: while args:
if args[0].startswith("-"): if args[0].startswith("-"):
......
...@@ -30,6 +30,8 @@ from .Errors import PyrexError, CompileError, error, warning ...@@ -30,6 +30,8 @@ from .Errors import PyrexError, CompileError, error, warning
from .Symtab import ModuleScope from .Symtab import ModuleScope
from .. import Utils from .. import Utils
from . import Options from . import Options
from .Options import CompilationOptions, default_options
from .CmdLine import parse_command_line
from . import Version # legacy import needed by old PyTables versions from . import Version # legacy import needed by old PyTables versions
version = Version.version # legacy attribute - use "Cython.__version__" instead version = Version.version # legacy attribute - use "Cython.__version__" instead
...@@ -82,6 +84,11 @@ class Context(object): ...@@ -82,6 +84,11 @@ class Context(object):
self.gdb_debug_outputwriter = None self.gdb_debug_outputwriter = None
@classmethod
def from_options(cls, options):
return cls(options.include_path, options.compiler_directives,
options.cplus, options.language_level, options=options)
def set_language_level(self, level): def set_language_level(self, level):
from .Future import print_function, unicode_literals, absolute_import, division, generator_stop from .Future import print_function, unicode_literals, absolute_import, division, generator_stop
future_directives = set() future_directives = set()
...@@ -449,7 +456,7 @@ def run_pipeline(source, options, full_module_name=None, context=None): ...@@ -449,7 +456,7 @@ def run_pipeline(source, options, full_module_name=None, context=None):
source_ext = os.path.splitext(source)[1] source_ext = os.path.splitext(source)[1]
options.configure_language_defaults(source_ext[1:]) # py/pyx options.configure_language_defaults(source_ext[1:]) # py/pyx
if context is None: if context is None:
context = options.create_context() context = Context.from_options(options)
# Set up source object # Set up source object
cwd = os.getcwd() cwd = os.getcwd()
...@@ -507,146 +514,6 @@ class CompilationSource(object): ...@@ -507,146 +514,6 @@ class CompilationSource(object):
self.cwd = cwd self.cwd = cwd
class CompilationOptions(object):
r"""
See default_options at the end of this module for a list of all possible
options and CmdLine.usage and CmdLine.parse_command_line() for their
meaning.
"""
def __init__(self, defaults=None, **kw):
self.include_path = []
if defaults:
if isinstance(defaults, CompilationOptions):
defaults = defaults.__dict__
else:
defaults = default_options
options = dict(defaults)
options.update(kw)
# let's assume 'default_options' contains a value for most known compiler options
# and validate against them
unknown_options = set(options) - set(default_options)
# ignore valid options that are not in the defaults
unknown_options.difference_update(['include_path'])
if unknown_options:
message = "got unknown compilation option%s, please remove: %s" % (
's' if len(unknown_options) > 1 else '',
', '.join(unknown_options))
raise ValueError(message)
directive_defaults = Options.get_directive_defaults()
directives = dict(options['compiler_directives']) # copy mutable field
# check for invalid directives
unknown_directives = set(directives) - set(directive_defaults)
if unknown_directives:
message = "got unknown compiler directive%s: %s" % (
's' if len(unknown_directives) > 1 else '',
', '.join(unknown_directives))
raise ValueError(message)
options['compiler_directives'] = directives
if directives.get('np_pythran', False) and not options['cplus']:
import warnings
warnings.warn("C++ mode forced when in Pythran mode!")
options['cplus'] = True
if 'language_level' in directives and 'language_level' not in kw:
options['language_level'] = directives['language_level']
elif not options.get('language_level'):
options['language_level'] = directive_defaults.get('language_level')
if 'formal_grammar' in directives and 'formal_grammar' not in kw:
options['formal_grammar'] = directives['formal_grammar']
if options['cache'] is True:
options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
self.__dict__.update(options)
def configure_language_defaults(self, source_extension):
if source_extension == 'py':
if self.compiler_directives.get('binding') is None:
self.compiler_directives['binding'] = True
def create_context(self):
return Context(self.include_path, self.compiler_directives,
self.cplus, self.language_level, options=self)
def get_fingerprint(self):
r"""
Return a string that contains all the options that are relevant for cache invalidation.
"""
# Collect only the data that can affect the generated file(s).
data = {}
for key, value in self.__dict__.items():
if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
# verbosity flags have no influence on the compilation result
continue
elif key in ['output_file', 'output_dir']:
# ignore the exact name of the output file
continue
elif key in ['timestamps']:
# the cache cares about the content of files, not about the timestamps of sources
continue
elif key in ['cache']:
# hopefully caching has no influence on the compilation result
continue
elif key in ['compiler_directives']:
# directives passed on to the C compiler do not influence the generated C code
continue
elif key in ['include_path']:
# this path changes which headers are tracked as dependencies,
# it has no influence on the generated C code
continue
elif key in ['working_path']:
# this path changes where modules and pxd files are found;
# their content is part of the fingerprint anyway, their
# absolute path does not matter
continue
elif key in ['create_extension']:
# create_extension() has already mangled the options, e.g.,
# embedded_metadata, when the fingerprint is computed so we
# ignore it here.
continue
elif key in ['build_dir']:
# the (temporary) directory where we collect dependencies
# has no influence on the C output
continue
elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
# all output files are contained in the cache so the types of
# files generated must be part of the fingerprint
data[key] = value
elif key in ['formal_grammar', 'evaluate_tree_assertions']:
# these bits can change whether compilation to C passes/fails
data[key] = value
elif key in ['embedded_metadata', 'emit_linenums', 'c_line_in_traceback', 'gdb_debug', 'relative_path_in_code_position_comments']:
# the generated code contains additional bits when these are set
data[key] = value
elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
# assorted bits that, e.g., influence the parser
data[key] = value
elif key == ['capi_reexport_cincludes']:
if self.capi_reexport_cincludes:
# our caching implementation does not yet include fingerprints of all the header files
raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
elif key == ['common_utility_include_dir']:
if self.common_utility_include_dir:
raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
else:
# any unexpected option should go into the fingerprint; it's better
# to recompile than to return incorrect results from the cache.
data[key] = value
def to_fingerprint(item):
r"""
Recursively turn item into a string, turning dicts into lists with
deterministic ordering.
"""
if isinstance(item, dict):
item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
return repr(item)
return to_fingerprint(data)
class CompilationResult(object): class CompilationResult(object):
""" """
Results from the Cython compiler: Results from the Cython compiler:
...@@ -709,7 +576,7 @@ def compile_multiple(sources, options): ...@@ -709,7 +576,7 @@ def compile_multiple(sources, options):
if these are specified in the options. if these are specified in the options.
""" """
# run_pipeline creates the context # run_pipeline creates the context
# context = options.create_context() # context = Context.from_options(options)
sources = [os.path.abspath(source) for source in sources] sources = [os.path.abspath(source) for source in sources]
processed = set() processed = set()
results = CompilationResultSet() results = CompilationResultSet()
...@@ -720,7 +587,7 @@ def compile_multiple(sources, options): ...@@ -720,7 +587,7 @@ def compile_multiple(sources, options):
for source in sources: for source in sources:
if source not in processed: if source not in processed:
if context is None: if context is None:
context = options.create_context() context = Context.from_options(options)
output_filename = get_output_filename(source, cwd, options) output_filename = get_output_filename(source, cwd, options)
out_of_date = context.c_file_out_of_date(source, output_filename) out_of_date = context.c_file_out_of_date(source, output_filename)
if (not timestamps) or out_of_date: if (not timestamps) or out_of_date:
...@@ -767,7 +634,6 @@ def main(command_line = 0): ...@@ -767,7 +634,6 @@ def main(command_line = 0):
args = sys.argv[1:] args = sys.argv[1:]
any_failures = 0 any_failures = 0
if command_line: if command_line:
from .CmdLine import parse_command_line
options, sources = parse_command_line(args) options, sources = parse_command_line(args)
else: else:
options = CompilationOptions(default_options) options = CompilationOptions(default_options)
...@@ -786,42 +652,3 @@ def main(command_line = 0): ...@@ -786,42 +652,3 @@ def main(command_line = 0):
any_failures = 1 any_failures = 1
if any_failures: if any_failures:
sys.exit(1) sys.exit(1)
# ------------------------------------------------------------------------
#
# Set the default options depending on the platform
#
# ------------------------------------------------------------------------
default_options = dict(
show_version = 0,
use_listing_file = 0,
errors_to_stderr = 1,
cplus = 0,
output_file = None,
annotate = None,
annotate_coverage_xml = None,
generate_pxi = 0,
capi_reexport_cincludes = 0,
working_path = "",
timestamps = None,
verbose = 0,
quiet = 0,
compiler_directives = {},
embedded_metadata = {},
evaluate_tree_assertions = False,
emit_linenums = False,
relative_path_in_code_position_comments = True,
c_line_in_traceback = True,
language_level = None, # warn but default to 2
formal_grammar = False,
gdb_debug = False,
compile_time_env = None,
common_utility_include_dir = None,
output_dir=None,
build_dir=None,
cache=None,
create_extension=None,
np_pythran=False
)
...@@ -4,6 +4,10 @@ ...@@ -4,6 +4,10 @@
from __future__ import absolute_import from __future__ import absolute_import
import os
from Cython import Utils
class ShouldBeFromDirective(object): class ShouldBeFromDirective(object):
...@@ -543,3 +547,184 @@ def parse_compile_time_env(s, current_settings=None): ...@@ -543,3 +547,184 @@ def parse_compile_time_env(s, current_settings=None):
name, value = [s.strip() for s in item.split('=', 1)] name, value = [s.strip() for s in item.split('=', 1)]
result[name] = parse_variable_value(value) result[name] = parse_variable_value(value)
return result return result
# ------------------------------------------------------------------------
# CompilationOptions are constructed from user input and are the `option`
# object passed throughout the compilation pipeline.
class CompilationOptions(object):
r"""
See default_options at the end of this module for a list of all possible
options and CmdLine.usage and CmdLine.parse_command_line() for their
meaning.
"""
def __init__(self, defaults=None, **kw):
self.include_path = []
if defaults:
if isinstance(defaults, CompilationOptions):
defaults = defaults.__dict__
else:
defaults = default_options
options = dict(defaults)
options.update(kw)
# let's assume 'default_options' contains a value for most known compiler options
# and validate against them
unknown_options = set(options) - set(default_options)
# ignore valid options that are not in the defaults
unknown_options.difference_update(['include_path'])
if unknown_options:
message = "got unknown compilation option%s, please remove: %s" % (
's' if len(unknown_options) > 1 else '',
', '.join(unknown_options))
raise ValueError(message)
directive_defaults = get_directive_defaults()
directives = dict(options['compiler_directives']) # copy mutable field
# check for invalid directives
unknown_directives = set(directives) - set(directive_defaults)
if unknown_directives:
message = "got unknown compiler directive%s: %s" % (
's' if len(unknown_directives) > 1 else '',
', '.join(unknown_directives))
raise ValueError(message)
options['compiler_directives'] = directives
if directives.get('np_pythran', False) and not options['cplus']:
import warnings
warnings.warn("C++ mode forced when in Pythran mode!")
options['cplus'] = True
if 'language_level' in directives and 'language_level' not in kw:
options['language_level'] = directives['language_level']
elif not options.get('language_level'):
options['language_level'] = directive_defaults.get('language_level')
if 'formal_grammar' in directives and 'formal_grammar' not in kw:
options['formal_grammar'] = directives['formal_grammar']
if options['cache'] is True:
options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
self.__dict__.update(options)
def configure_language_defaults(self, source_extension):
if source_extension == 'py':
if self.compiler_directives.get('binding') is None:
self.compiler_directives['binding'] = True
def get_fingerprint(self):
r"""
Return a string that contains all the options that are relevant for cache invalidation.
"""
# Collect only the data that can affect the generated file(s).
data = {}
for key, value in self.__dict__.items():
if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
# verbosity flags have no influence on the compilation result
continue
elif key in ['output_file', 'output_dir']:
# ignore the exact name of the output file
continue
elif key in ['timestamps']:
# the cache cares about the content of files, not about the timestamps of sources
continue
elif key in ['cache']:
# hopefully caching has no influence on the compilation result
continue
elif key in ['compiler_directives']:
# directives passed on to the C compiler do not influence the generated C code
continue
elif key in ['include_path']:
# this path changes which headers are tracked as dependencies,
# it has no influence on the generated C code
continue
elif key in ['working_path']:
# this path changes where modules and pxd files are found;
# their content is part of the fingerprint anyway, their
# absolute path does not matter
continue
elif key in ['create_extension']:
# create_extension() has already mangled the options, e.g.,
# embedded_metadata, when the fingerprint is computed so we
# ignore it here.
continue
elif key in ['build_dir']:
# the (temporary) directory where we collect dependencies
# has no influence on the C output
continue
elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
# all output files are contained in the cache so the types of
# files generated must be part of the fingerprint
data[key] = value
elif key in ['formal_grammar', 'evaluate_tree_assertions']:
# these bits can change whether compilation to C passes/fails
data[key] = value
elif key in ['embedded_metadata', 'emit_linenums',
'c_line_in_traceback', 'gdb_debug',
'relative_path_in_code_position_comments']:
# the generated code contains additional bits when these are set
data[key] = value
elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
# assorted bits that, e.g., influence the parser
data[key] = value
elif key == ['capi_reexport_cincludes']:
if self.capi_reexport_cincludes:
# our caching implementation does not yet include fingerprints of all the header files
raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
elif key == ['common_utility_include_dir']:
if self.common_utility_include_dir:
raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
else:
# any unexpected option should go into the fingerprint; it's better
# to recompile than to return incorrect results from the cache.
data[key] = value
def to_fingerprint(item):
r"""
Recursively turn item into a string, turning dicts into lists with
deterministic ordering.
"""
if isinstance(item, dict):
item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
return repr(item)
return to_fingerprint(data)
# ------------------------------------------------------------------------
#
# Set the default options depending on the platform
#
# ------------------------------------------------------------------------
default_options = dict(
show_version=0,
use_listing_file=0,
errors_to_stderr=1,
cplus=0,
output_file=None,
annotate=None,
annotate_coverage_xml=None,
generate_pxi=0,
capi_reexport_cincludes=0,
working_path="",
timestamps=None,
verbose=0,
quiet=0,
compiler_directives={},
embedded_metadata={},
evaluate_tree_assertions=False,
emit_linenums=False,
relative_path_in_code_position_comments=True,
c_line_in_traceback=True,
language_level=None, # warn but default to 2
formal_grammar=False,
gdb_debug=False,
compile_time_env=None,
common_utility_include_dir=None,
output_dir=None,
build_dir=None,
cache=None,
create_extension=None,
np_pythran=False
)
...@@ -3,7 +3,7 @@ import os ...@@ -3,7 +3,7 @@ import os
from Cython.TestUtils import TransformTest from Cython.TestUtils import TransformTest
from Cython.Compiler.ParseTreeTransforms import * from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Nodes import * from Cython.Compiler.Nodes import *
from Cython.Compiler import Main, Symtab from Cython.Compiler import Main, Symtab, Options
class TestNormalizeTree(TransformTest): class TestNormalizeTree(TransformTest):
...@@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest): ...@@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest):
def setUp(self): def setUp(self):
super(TestInterpretCompilerDirectives, self).setUp() super(TestInterpretCompilerDirectives, self).setUp()
compilation_options = Main.CompilationOptions(Main.default_options) compilation_options = Options.CompilationOptions(Options.default_options)
ctx = compilation_options.create_context() ctx = Main.Context.from_options(compilation_options)
transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
......
...@@ -11,7 +11,7 @@ from contextlib import contextmanager ...@@ -11,7 +11,7 @@ from contextlib import contextmanager
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives
from Cython.Compiler import Main, Symtab, Visitor from Cython.Compiler import Main, Symtab, Visitor, Options
from Cython.TestUtils import TransformTest from Cython.TestUtils import TransformTest
TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools')) TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools'))
...@@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper): ...@@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper):
""" """
def setUp(self): def setUp(self):
super(TestTypeInjection, self).setUp() super(TestTypeInjection, self).setUp()
compilation_options = Main.CompilationOptions(Main.default_options) compilation_options = Options.CompilationOptions(Options.default_options)
ctx = compilation_options.create_context() ctx = Main.Context.from_options(compilation_options)
transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
self.declarations_finder = DeclarationsFinder() self.declarations_finder = DeclarationsFinder()
......
...@@ -1029,9 +1029,9 @@ class CythonCompileTestCase(unittest.TestCase): ...@@ -1029,9 +1029,9 @@ class CythonCompileTestCase(unittest.TestCase):
try: try:
CompilationOptions CompilationOptions
except NameError: except NameError:
from Cython.Compiler.Main import CompilationOptions from Cython.Compiler.Options import CompilationOptions
from Cython.Compiler.Main import compile as cython_compile from Cython.Compiler.Main import compile as cython_compile
from Cython.Compiler.Main import default_options from Cython.Compiler.Options import default_options
common_utility_include_dir = self.common_utility_dir common_utility_include_dir = self.common_utility_dir
options = CompilationOptions( options = CompilationOptions(
...@@ -2232,7 +2232,7 @@ def time_stamper_thread(interval=10): ...@@ -2232,7 +2232,7 @@ def time_stamper_thread(interval=10):
def configure_cython(options): def configure_cython(options):
global CompilationOptions, pyrex_default_options, cython_compile global CompilationOptions, pyrex_default_options, cython_compile
from Cython.Compiler.Main import \ from Cython.Compiler.Options import \
CompilationOptions, \ CompilationOptions, \
default_options as pyrex_default_options default_options as pyrex_default_options
from Cython.Compiler.Options import _directive_defaults as directive_defaults from Cython.Compiler.Options import _directive_defaults as directive_defaults
...@@ -2304,7 +2304,7 @@ def runtests(options, cmd_args, coverage=None): ...@@ -2304,7 +2304,7 @@ def runtests(options, cmd_args, coverage=None):
options.cleanup_sharedlibs = False options.cleanup_sharedlibs = False
options.fork = False options.fork = False
if WITH_CYTHON and include_debugger: if WITH_CYTHON and include_debugger:
from Cython.Compiler.Main import default_options as compiler_default_options from Cython.Compiler.Options import default_options as compiler_default_options
compiler_default_options['gdb_debug'] = True compiler_default_options['gdb_debug'] = True
compiler_default_options['output_dir'] = os.getcwd() compiler_default_options['output_dir'] = os.getcwd()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment