Code.py 86.4 KB
Newer Older
1
# cython: language_level = 2
William Stein's avatar
William Stein committed
2
#
3
#   Code output module
William Stein's avatar
William Stein committed
4 5
#

6 7
from __future__ import absolute_import

Stefan Behnel's avatar
Stefan Behnel committed
8
import cython
9 10
cython.declare(os=object, re=object, operator=object,
               Naming=object, Options=object, StringEncoding=object,
Stefan Behnel's avatar
Stefan Behnel committed
11
               Utils=object, SourceDescriptor=object, StringIOTree=object,
12
               DebugFlags=object, basestring=object)
Stefan Behnel's avatar
Stefan Behnel committed
13

14
import os
15
import re
16
import shutil
Stefan Behnel's avatar
Stefan Behnel committed
17
import sys
18
import operator
19
import textwrap
20 21
from string import Template
from functools import partial
Stefan Behnel's avatar
Stefan Behnel committed
22 23
from contextlib import closing
from collections import defaultdict
24

Robert Bradshaw's avatar
Robert Bradshaw committed
25 26 27 28 29
try:
    import hashlib
except ImportError:
    import md5 as hashlib

30 31 32 33
from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
34
from . import Version
35 36 37
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree
38

Stefan Behnel's avatar
Stefan Behnel committed
39
try:
Stefan Behnel's avatar
Stefan Behnel committed
40 41 42
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring
43

Stefan Behnel's avatar
Stefan Behnel committed
44
KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7)
Stefan Behnel's avatar
Stefan Behnel committed
45

46 47

non_portable_builtins_map = {
48
    # builtins that have different names in different Python versions
49 50
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
51
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
52
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
53
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
54
}
55

56 57
basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
58 59
    'PyTypeObject': 'PyHeapTypeObject',
}
60

61 62 63 64
uncachable_builtins = [
    # builtin names that cannot be cached because they may or may not
    # be available at import time
    'WindowsError',
65 66
    '_',  # e.g. gettext
]
67

68 69 70 71 72 73 74
special_py_methods = set([
    '__cinit__', '__dealloc__', '__richcmp__', '__next__',
    '__await__', '__aiter__', '__anext__',
    '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__',
    '__getcharbuffer__', '__getbuffer__', '__releasebuffer__'
])

75 76 77 78
modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get

79 80
is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match

Stefan Behnel's avatar
Stefan Behnel committed
81

82 83 84 85 86
def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")
Mark Florisson's avatar
Mark Florisson committed
87

Stefan Behnel's avatar
Stefan Behnel committed
88

89
class UtilityCodeBase(object):
90 91 92 93 94 95
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####
96

97
        [proto declarations]
98

99
        ##### MyUtility.init #####
100

101
        [code run at module initialization]
102 103 104

        ##### MyUtility #####
        #@requires: MyOtherUtility
105
        #@substitute: naming
106

107
        [definitions]
108 109 110 111 112 113 114 115 116

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """
117 118

    is_cython_utility = False
119
    requires = None
120 121
    _utility_cache = {}

122
    @classmethod
123
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
124 125
        if utility is None:
            return
126

127 128 129 130 131
        code = '\n'.join(lines)
        if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
            del tags['substitute']
            try:
                code = Template(code).substitute(vars(Naming))
132
            except (KeyError, ValueError) as e:
133 134 135 136 137
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code
138

139
        if type == 'proto':
140
            utility[0] = code
141 142 143
        elif type.startswith('proto.'):
            utility[0] = code
            utility[1] = type[6:]
144
        elif type == 'impl':
145
            utility[2] = code
146
        else:
147
            all_tags = utility[3]
Stefan Behnel's avatar
Stefan Behnel committed
148 149
            if KEYWORDS_MUST_BE_BYTES:
                type = type.encode('ASCII')
150
            all_tags[type] = code
151

152
        if tags:
153
            all_tags = utility[3]
Stefan Behnel's avatar
Stefan Behnel committed
154
            for name, values in tags.items():
Stefan Behnel's avatar
Stefan Behnel committed
155 156
                if KEYWORDS_MUST_BE_BYTES:
                    name = name.encode('ASCII')
157
                all_tags.setdefault(name, set()).update(values)
158

159
    @classmethod
160 161 162 163 164
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

165
        filename = os.path.join(get_utility_dir(), path)
166 167 168
        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
Stefan Behnel's avatar
Stefan Behnel committed
169
            strip_comments = partial(re.compile(r'^\s*#.*').sub, '')
Stefan Behnel's avatar
Stefan Behnel committed
170
            rstrip = StringEncoding._unicode.rstrip
171
        else:
172
            comment = '/'
Stefan Behnel's avatar
Stefan Behnel committed
173
            strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '')
174
            rstrip = partial(re.compile(r'\s+(\\?)$').sub, r'\1')
175
        match_special = re.compile(
176
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
Stefan Behnel's avatar
Stefan Behnel committed
177 178
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') %
            {'C': comment}).match
179
        match_type = re.compile('(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match
180

Stefan Behnel's avatar
Stefan Behnel committed
181
        with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f:
182
            all_lines = f.readlines()
183

184
        utilities = defaultdict(lambda: [None, None, None, {}])
185
        lines = []
Stefan Behnel's avatar
Stefan Behnel committed
186
        tags = defaultdict(set)
187 188 189
        utility = type = None
        begin_lineno = 0

Mark Florisson's avatar
Mark Florisson committed
190
        for lineno, line in enumerate(all_lines):
191
            m = match_special(line)
Mark Florisson's avatar
Mark Florisson committed
192
            if m:
193
                if m.group('name'):
194 195 196 197 198 199
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1
                    del lines[:]
                    tags.clear()

200 201 202 203
                    name = m.group('name')
                    mtype = match_type(name)
                    if mtype:
                        name, type = mtype.groups()
204
                    else:
205
                        type = 'impl'
Stefan Behnel's avatar
Stefan Behnel committed
206
                    utility = utilities[name]
Mark Florisson's avatar
Mark Florisson committed
207
                else:
Stefan Behnel's avatar
Stefan Behnel committed
208 209
                    tags[m.group('tag')].add(m.group('value'))
                    lines.append('')  # keep line number correct
Mark Florisson's avatar
Mark Florisson committed
210
            else:
Stefan Behnel's avatar
Stefan Behnel committed
211
                lines.append(rstrip(strip_comments(line)))
Mark Florisson's avatar
Mark Florisson committed
212

213
        if utility is None:
214 215 216
            raise ValueError("Empty utility code file")

        # Don't forget to add the last utility code
217
        cls._add_utility(utility, type, lines, begin_lineno, tags)
218

Stefan Behnel's avatar
Stefan Behnel committed
219
        utilities = dict(utilities)  # un-defaultdict-ify
220 221 222
        cls._utility_cache[path] = utilities
        return utilities

223
    @classmethod
224
    def load(cls, util_code_name, from_file=None, **kwargs):
225
        """
226 227 228 229
        Load utility code from a file specified by from_file (relative to
        Cython/Utility) and name util_code_name.  If from_file is not given,
        load it from the file util_code_name.*.  There should be only one
        file matched by this pattern.
230
        """
231 232 233
        if '::' in util_code_name:
            from_file, util_code_name = util_code_name.rsplit('::', 1)
        if not from_file:
234 235
            utility_dir = get_utility_dir()
            prefix = util_code_name + '.'
236 237 238 239 240 241 242 243 244
            try:
                listing = os.listdir(utility_dir)
            except OSError:
                # XXX the code below assumes as 'zipimport.zipimporter' instance
                # XXX should be easy to generalize, but too lazy right now to write it
                import zipfile
                global __loader__
                loader = __loader__
                archive = loader.archive
Stefan Behnel's avatar
Stefan Behnel committed
245
                with closing(zipfile.ZipFile(archive)) as fileobj:
246 247 248 249 250
                    listing = [os.path.basename(name)
                               for name in fileobj.namelist()
                               if os.path.join(archive, name).startswith(utility_dir)]
            files = [filename for filename in listing
                     if filename.startswith(prefix)]
251 252 253 254 255
            if not files:
                raise ValueError("No match found for utility code " + util_code_name)
            if len(files) > 1:
                raise ValueError("More than one filename match found for utility code " + util_code_name)
            from_file = files[0]
256 257

        utilities = cls.load_utilities_from_file(from_file)
258
        proto, proto_block, impl, tags = utilities[util_code_name]
259 260 261

        if tags:
            orig_kwargs = kwargs.copy()
Stefan Behnel's avatar
Stefan Behnel committed
262
            for name, values in tags.items():
263 264 265 266
                if name in kwargs:
                    continue
                # only pass lists when we have to: most argument expect one value or None
                if name == 'requires':
267 268 269 270 271 272 273
                    if orig_kwargs:
                        values = [cls.load(dep, from_file, **orig_kwargs)
                                  for dep in sorted(values)]
                    else:
                        # dependencies are rarely unique, so use load_cached() when we can
                        values = [cls.load_cached(dep, from_file)
                                  for dep in sorted(values)]
274 275 276 277 278
                elif not values:
                    values = None
                elif len(values) == 1:
                    values = values[0]
                kwargs[name] = values
Mark Florisson's avatar
Mark Florisson committed
279

280 281
        if proto is not None:
            kwargs['proto'] = proto
282 283
        if proto_block is not None:
            kwargs['proto_block'] = proto_block
284 285
        if impl is not None:
            kwargs['impl'] = impl
Mark Florisson's avatar
Mark Florisson committed
286

287 288
        if 'name' not in kwargs:
            kwargs['name'] = util_code_name
Mark Florisson's avatar
Mark Florisson committed
289

290 291 292 293 294
        if 'file' not in kwargs and from_file:
            kwargs['file'] = from_file
        return cls(**kwargs)

    @classmethod
295
    def load_cached(cls, utility_code_name, from_file=None, __cache={}):
296
        """
297
        Calls .load(), but using a per-type cache based on utility name and file name.
298
        """
299 300
        key = (cls, from_file, utility_code_name)
        try:
301
            return __cache[key]
302 303
        except KeyError:
            pass
304
        code = __cache[key] = cls.load(utility_code_name, from_file)
305
        return code
306

307 308 309 310 311 312 313
    @classmethod
    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
        """
        Load a utility code as a string. Returns (proto, implementation)
        """
        util = cls.load(util_code_name, from_file, **kwargs)
        proto, impl = util.proto, util.impl
314
        return util.format_code(proto), util.format_code(impl)
315

316
    def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
317 318 319 320 321 322 323
        """
        Format a code section for output.
        """
        if code_string:
            code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'
        return code_string

324
    def __str__(self):
325
        return "<%s(%s)>" % (type(self).__name__, self.name)
326

327
    def get_tree(self, **kwargs):
328
        pass
329

Stefan Behnel's avatar
Stefan Behnel committed
330

331
class UtilityCode(UtilityCodeBase):
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
    """
    Stores utility code to add during code generation.

    See GlobalState.put_utility_code.

    hashes/equals by instance

    proto           C prototypes
    impl            implemenation code
    init            code to call on module initialization
    requires        utility code dependencies
    proto_block     the place in the resulting file where the prototype should
                    end up
    name            name of the utility code (or None)
    file            filename of the utility code file this utility was loaded
                    from (or None)
    """
349

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
350
    def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
351
                 proto_block='utility_code_proto', name=None, file=None):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
352
        # proto_block: Which code block to dump prototype in. See GlobalState.
353 354 355 356 357 358 359
        self.proto = proto
        self.impl = impl
        self.init = init
        self.cleanup = cleanup
        self.requires = requires
        self._cache = {}
        self.specialize_list = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
360
        self.proto_block = proto_block
361
        self.name = name
362
        self.file = file
363

364 365 366 367 368 369
    def __hash__(self):
        return hash((self.proto, self.impl))

    def __eq__(self, other):
        if self is other:
            return True
370 371
        self_type, other_type = type(self), type(other)
        if self_type is not other_type and not (isinstance(other, self_type) or isinstance(self, other_type)):
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
            return False

        self_proto = getattr(self, 'proto', None)
        other_proto = getattr(other, 'proto', None)
        return (self_proto, self.impl) == (other_proto, other.impl)

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return s % context

    def specialize(self, pyrex_type=None, **data):
387 388
        # Dicts aren't hashable...
        if pyrex_type is not None:
389
            data['type'] = pyrex_type.empty_declaration_code()
Craig Citro's avatar
Craig Citro committed
390
            data['type_name'] = pyrex_type.specialization_name()
Stefan Behnel's avatar
Stefan Behnel committed
391
        key = tuple(sorted(data.items()))
392 393 394 395 396 397 398
        try:
            return self._cache[key]
        except KeyError:
            if self.requires is None:
                requires = None
            else:
                requires = [r.specialize(data) for r in self.requires]
399

400
            s = self._cache[key] = UtilityCode(
Stefan Behnel's avatar
Stefan Behnel committed
401 402 403 404 405 406
                self.none_or_sub(self.proto, data),
                self.none_or_sub(self.impl, data),
                self.none_or_sub(self.init, data),
                self.none_or_sub(self.cleanup, data),
                requires,
                self.proto_block)
407

408 409 410
            self.specialize_list.append(s)
            return s

411 412 413
    def inject_string_constants(self, impl, output):
        """Replace 'PYIDENT("xyz")' by a constant Python identifier cname.
        """
414 415 416
        if 'PYIDENT(' not in impl:
            return False, impl

417 418 419 420 421 422 423 424 425 426
        replacements = {}
        def externalise(matchobj):
            name = matchobj.group(1)
            try:
                cname = replacements[name]
            except KeyError:
                cname = replacements[name] = output.get_interned_identifier(
                    StringEncoding.EncodedString(name)).cname
            return cname

427 428
        impl = re.sub(r'PYIDENT\("([^"]+)"\)', externalise, impl)
        assert 'PYIDENT(' not in impl
429
        return bool(replacements), impl
430

431 432 433 434 435 436
    def inject_unbound_methods(self, impl, output):
        """Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname.
        """
        if 'CALL_UNBOUND_METHOD(' not in impl:
            return False, impl

437
        utility_code = set()
438 439 440 441 442
        def externalise(matchobj):
            type_cname, method_name, args = matchobj.groups()
            args = [arg.strip() for arg in args[1:].split(',')]
            if len(args) == 1:
                call = '__Pyx_CallUnboundCMethod0'
443
                utility_code.add("CallUnboundCMethod0")
444 445
            elif len(args) == 2:
                call = '__Pyx_CallUnboundCMethod1'
446
                utility_code.add("CallUnboundCMethod1")
447 448 449
            else:
                assert False, "CALL_UNBOUND_METHOD() requires 1 or 2 call arguments"

450
            cname = output.get_cached_unbound_method(type_cname, method_name, len(args))
451 452
            return '%s(&%s, %s)' % (call, cname, ', '.join(args))

453
        impl = re.sub(r'CALL_UNBOUND_METHOD\(([a-zA-Z_]+),\s*"([^"]+)"((?:,\s*[^),]+)+)\)', externalise, impl)
454
        assert 'CALL_UNBOUND_METHOD(' not in impl
455 456 457 458

        for helper in sorted(utility_code):
            output.use_utility_code(UtilityCode.load_cached(helper, "ObjectHandling.c"))
        return bool(utility_code), impl
459

460 461 462 463 464 465 466 467 468 469 470 471
    def wrap_c_strings(self, impl):
        """Replace CSTRING('''xyz''') by a C compatible string
        """
        if 'CSTRING(' not in impl:
            return impl

        def split_string(matchobj):
            content = matchobj.group(1).replace('"', '\042')
            return ''.join(
                '"%s\\n"\n' % line if not line.endswith('\\') or line.endswith('\\\\') else '"%s"\n' % line[:-1]
                for line in content.splitlines())

472
        impl = re.sub(r'CSTRING\(\s*"""([^"]*(?:"[^"]+)*)"""\s*\)', split_string, impl)
473 474 475
        assert 'CSTRING(' not in impl
        return impl

476 477 478 479 480
    def put_code(self, output):
        if self.requires:
            for dependency in self.requires:
                output.use_utility_code(dependency)
        if self.proto:
481 482 483 484
            writer = output[self.proto_block]
            writer.putln("/* %s.proto */" % self.name)
            writer.put_or_include(
                self.format_code(self.proto), '%s_proto' % self.name)
485
        if self.impl:
486
            impl = self.format_code(self.wrap_c_strings(self.impl))
487 488
            is_specialised1, impl = self.inject_string_constants(impl, output)
            is_specialised2, impl = self.inject_unbound_methods(impl, output)
489
            writer = output['utility_code_def']
Stefan Behnel's avatar
Stefan Behnel committed
490
            writer.putln("/* %s */" % self.name)
491
            if not (is_specialised1 or is_specialised2):
492
                # no module specific adaptations => can be reused
493
                writer.put_or_include(impl, '%s_impl' % self.name)
494
            else:
495
                writer.put(impl)
496 497
        if self.init:
            writer = output['init_globals']
498
            writer.putln("/* %s.init */" % self.name)
499
            if isinstance(self.init, basestring):
500
                writer.put(self.format_code(self.init))
501 502
            else:
                self.init(writer, output.module_pos)
503 504
            writer.putln(writer.error_goto_if_PyErr(output.module_pos))
            writer.putln()
505 506
        if self.cleanup and Options.generate_cleanup_code:
            writer = output['cleanup_globals']
507
            writer.putln("/* %s.cleanup */" % self.name)
508
            if isinstance(self.cleanup, basestring):
509 510 511
                writer.put_or_include(
                    self.format_code(self.cleanup),
                    '%s_cleanup' % self.name)
512 513
            else:
                self.cleanup(writer, output.module_pos)
514 515


516
def sub_tempita(s, context, file=None, name=None):
517 518 519
    "Run tempita on string s with given context."
    if not s:
        return None
520

521 522 523 524
    if file:
        context['__name'] = "%s:%s" % (file, name)
    elif name:
        context['__name'] = name
525

526
    from ..Tempita import sub
527
    return sub(s, **context)
528

529

530
class TempitaUtilityCode(UtilityCode):
531
    def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
532 533
        if context is None:
            context = {}
534 535
        proto = sub_tempita(proto, context, file, name)
        impl = sub_tempita(impl, context, file, name)
536
        init = sub_tempita(init, context, file, name)
537
        super(TempitaUtilityCode, self).__init__(
538
            proto, impl, init=init, name=name, file=file, **kwargs)
539

540 541 542 543 544 545 546 547 548 549 550 551
    @classmethod
    def load_cached(cls, utility_code_name, from_file=None, context=None, __cache={}):
        context_key = tuple(sorted(context.items())) if context else None
        assert hash(context_key) is not None  # raise TypeError if not hashable
        key = (cls, from_file, utility_code_name, context_key)
        try:
            return __cache[key]
        except KeyError:
            pass
        code = __cache[key] = cls.load(utility_code_name, from_file, context=context)
        return code

552 553 554 555 556 557 558
    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return sub_tempita(s, context, self.file, self.name)
559 560 561 562 563 564 565


class LazyUtilityCode(UtilityCodeBase):
    """
    Utility code that calls a callback with the root code writer when
    available. Useful when you only have 'env' but not 'code'.
    """
566
    __name__ = '<lazy>'
567 568 569 570 571 572 573 574

    def __init__(self, callback):
        self.callback = callback

    def put_code(self, globalstate):
        utility = self.callback(globalstate.rootwriter)
        globalstate.use_utility_code(utility)

575

576 577 578 579 580 581 582 583 584
class FunctionState(object):
    # return_label     string          function return point label
    # error_label      string          error catch point label
    # continue_label   string          loop continue point label
    # break_label      string          loop break point label
    # return_from_error_cleanup_label string
    # label_counter    integer         counter for naming labels
    # in_try_finally   boolean         inside try of try...finally
    # exc_vars         (string * 3)    exception variables for reraise, or None
585
    # can_trace        boolean         line tracing is supported in the current context
586
    # scope            Scope           the scope object of the current function
587

588
    # Not used for now, perhaps later
589
    def __init__(self, owner, names_taken=set(), scope=None):
590
        self.names_taken = names_taken
591
        self.owner = owner
592
        self.scope = scope
593

594 595
        self.error_label = None
        self.label_counter = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
596
        self.labels_used = set()
597 598 599 600
        self.return_label = self.new_label()
        self.new_error_label()
        self.continue_label = None
        self.break_label = None
601
        self.yield_labels = []
602

603 604
        self.in_try_finally = 0
        self.exc_vars = None
605
        self.can_trace = False
606
        self.gil_owned = True
607

608
        self.temps_allocated = [] # of (name, type, manage_ref, static)
609 610
        self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
        self.temps_used_type = {} # name -> (type, manage_ref)
611
        self.temp_counter = 0
612
        self.closure_temps = None
613

614 615 616 617
        # This is used to collect temporaries, useful to find out which temps
        # need to be privatized in parallel sections
        self.collect_temps_stack = []

618 619 620 621 622
        # This is used for the error indicator, which needs to be local to the
        # function. It used to be global, which relies on the GIL being held.
        # However, exceptions may need to be propagated through 'nogil'
        # sections, in which case we introduce a race condition.
        self.should_declare_error_indicator = False
623
        self.uses_error_indicator = False
624

625 626
    # labels

627
    def new_label(self, name=None):
628 629
        n = self.label_counter
        self.label_counter = n + 1
630 631 632 633
        label = "%s%d" % (Naming.label_prefix, n)
        if name is not None:
            label += '_' + name
        return label
634

635 636 637 638 639 640
    def new_yield_label(self):
        label = self.new_label('resume_from_yield')
        num_and_label = (len(self.yield_labels) + 1, label)
        self.yield_labels.append(num_and_label)
        return num_and_label

641 642
    def new_error_label(self):
        old_err_lbl = self.error_label
643
        self.error_label = self.new_label('error')
644
        return old_err_lbl
645

646 647 648 649
    def get_loop_labels(self):
        return (
            self.continue_label,
            self.break_label)
650

651 652 653
    def set_loop_labels(self, labels):
        (self.continue_label,
         self.break_label) = labels
654

655 656 657
    def new_loop_labels(self):
        old_labels = self.get_loop_labels()
        self.set_loop_labels(
658
            (self.new_label("continue"),
Robert Bradshaw's avatar
Robert Bradshaw committed
659
             self.new_label("break")))
660
        return old_labels
661

662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
    def get_all_labels(self):
        return (
            self.continue_label,
            self.break_label,
            self.return_label,
            self.error_label)

    def set_all_labels(self, labels):
        (self.continue_label,
         self.break_label,
         self.return_label,
         self.error_label) = labels

    def all_new_labels(self):
        old_labels = self.get_all_labels()
        new_labels = []
678
        for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
679
            if old_label:
680
                new_labels.append(self.new_label(name))
681 682 683 684
            else:
                new_labels.append(old_label)
        self.set_all_labels(new_labels)
        return old_labels
685

686
    def use_label(self, lbl):
Stefan Behnel's avatar
Stefan Behnel committed
687
        self.labels_used.add(lbl)
688

689 690 691
    def label_used(self, lbl):
        return lbl in self.labels_used

692 693
    # temp handling

694
    def allocate_temp(self, type, manage_ref, static=False):
695 696 697 698 699
        """
        Allocates a temporary (which may create a new one or get a previously
        allocated and released one of the same type). Type is simply registered
        and handed back, but will usually be a PyrexType.

700 701 702 703 704
        If type.is_pyobject, manage_ref comes into play. If manage_ref is set to
        True, the temp will be decref-ed on return statements and in exception
        handling clauses. Otherwise the caller has to deal with any reference
        counting of the variable.

705 706 707 708
        If not type.is_pyobject, then manage_ref will be ignored, but it
        still has to be passed. It is recommended to pass False by convention
        if it is known that type will never be a Python object.

709 710 711 712
        static=True marks the temporary declaration with "static".
        This is only used when allocating backing store for a module-level
        C array literals.

713 714
        A C string referring to the variable is returned.
        """
715
        if type.is_const and not type.is_reference:
716
            type = type.const_base_type
717
        elif type.is_reference and not type.is_fake_reference:
718
            type = type.ref_base_type
719
        if not type.is_pyobject and not type.is_memoryviewslice:
720 721 722
            # Make manage_ref canonical, so that manage_ref will always mean
            # a decref is needed.
            manage_ref = False
723

724
        freelist = self.temps_free.get((type, manage_ref))
725 726 727
        if freelist is not None and freelist[0]:
            result = freelist[0].pop()
            freelist[1].remove(result)
728
        else:
729 730 731
            while True:
                self.temp_counter += 1
                result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
732
                if result not in self.names_taken: break
733
            self.temps_allocated.append((result, type, manage_ref, static))
734
        self.temps_used_type[result] = (type, manage_ref)
735
        if DebugFlags.debug_temp_code_comments:
736
            self.owner.putln("/* %s allocated (%s) */" % (result, type))
737 738 739 740

        if self.collect_temps_stack:
            self.collect_temps_stack[-1].add((result, type))

741 742 743 744 745 746 747
        return result

    def release_temp(self, name):
        """
        Releases a temporary so that it can be reused by other code needing
        a temp of the same type.
        """
748 749
        type, manage_ref = self.temps_used_type[name]
        freelist = self.temps_free.get((type, manage_ref))
750
        if freelist is None:
751
            freelist = ([], set())  # keep order in list and make lookups in set fast
752
            self.temps_free[(type, manage_ref)] = freelist
753
        if name in freelist[1]:
754
            raise RuntimeError("Temp %s freed twice!" % name)
755 756
        freelist[0].append(name)
        freelist[1].add(name)
757 758
        if DebugFlags.debug_temp_code_comments:
            self.owner.putln("/* %s released */" % name)
759

760
    def temps_in_use(self):
761
        """Return a list of (cname,type,manage_ref) tuples of temp names and their type
762 763 764
        that are currently in use.
        """
        used = []
765
        for name, type, manage_ref, static in self.temps_allocated:
766
            freelist = self.temps_free.get((type, manage_ref))
767
            if freelist is None or name not in freelist[1]:
768
                used.append((name, type, manage_ref and type.is_pyobject))
769 770
        return used

771
    def temps_holding_reference(self):
772
        """Return a list of (cname,type) tuples of temp names and their type
773 774
        that are currently in use. This includes only temps of a
        Python object type which owns its reference.
775 776
        """
        return [(name, type)
777
                for name, type, manage_ref in self.temps_in_use()
778
                if manage_ref  and type.is_pyobject]
779 780 781 782 783

    def all_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python objects.
        """
        return [(cname, type)
Stefan Behnel's avatar
Stefan Behnel committed
784 785
                for cname, type, manage_ref, static in self.temps_allocated
                if manage_ref]
786

787 788 789 790 791 792 793
    def all_free_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python
        objects that are not currently in use.  This is used by
        try-except and try-finally blocks to clean up temps in the
        error case.
        """
        return [(cname, type)
794
                for (type, manage_ref), freelist in self.temps_free.items() if manage_ref
795
                for cname in freelist[0]]
796

797 798 799 800
    def start_collecting_temps(self):
        """
        Useful to find out which temps were used in a code block
        """
Robert Bradshaw's avatar
Robert Bradshaw committed
801
        self.collect_temps_stack.append(set())
802 803 804

    def stop_collecting_temps(self):
        return self.collect_temps_stack.pop()
805

806 807 808
    def init_closure_temps(self, scope):
        self.closure_temps = ClosureTempAllocator(scope)

809

Stefan Behnel's avatar
Stefan Behnel committed
810 811 812 813 814 815 816
class NumConst(object):
    """Global info about a Python number constant held by GlobalState.

    cname       string
    value       string
    py_type     string     int, long, float
    value_code  string     evaluation code if different from value
817 818
    """

Stefan Behnel's avatar
Stefan Behnel committed
819
    def __init__(self, cname, value, py_type, value_code=None):
820 821
        self.cname = cname
        self.value = value
Stefan Behnel's avatar
Stefan Behnel committed
822 823 824
        self.py_type = py_type
        self.value_code = value_code or value

825

826 827 828 829 830 831 832 833 834 835
class PyObjectConst(object):
    """Global info about a generic constant held by GlobalState.
    """
    # cname       string
    # type        PyrexType

    def __init__(self, cname, type):
        self.cname = cname
        self.type = type

Stefan Behnel's avatar
Stefan Behnel committed
836

837
cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
838
               replace_identifier=object, find_alphanums=object)
839
possible_unicode_identifier = re.compile(br"(?![0-9])\w+$".decode('ascii'), re.U).match
Stefan Behnel's avatar
Stefan Behnel committed
840
possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
841
replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
842
find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
843 844 845 846 847 848 849 850 851 852 853 854 855

class StringConst(object):
    """Global info about a C string constant held by GlobalState.
    """
    # cname            string
    # text             EncodedString or BytesLiteral
    # py_strings       {(identifier, encoding) : PyStringConst}

    def __init__(self, cname, text, byte_string):
        self.cname = cname
        self.text = text
        self.escaped_value = StringEncoding.escape_byte_string(byte_string)
        self.py_strings = None
856 857 858
        self.py_versions = []

    def add_py_version(self, version):
859
        if not version:
Stefan Behnel's avatar
Stefan Behnel committed
860
            self.py_versions = [2, 3]
861
        elif version not in self.py_versions:
862
            self.py_versions.append(version)
863

864 865
    def get_py_string_const(self, encoding, identifier=None,
                            is_str=False, py3str_cstring=None):
866 867 868
        py_strings = self.py_strings
        text = self.text

Stefan Behnel's avatar
Stefan Behnel committed
869
        is_str = bool(identifier or is_str)
870
        is_unicode = encoding is None and not is_str
871

872 873 874 875 876 877 878 879 880 881 882 883
        if encoding is None:
            # unicode string
            encoding_key = None
        else:
            # bytes or str
            encoding = encoding.lower()
            if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
                encoding = None
                encoding_key = None
            else:
                encoding_key = ''.join(find_alphanums(encoding))

884 885 886 887 888 889
        key = (is_str, is_unicode, encoding_key, py3str_cstring)
        if py_strings is not None:
            try:
                return py_strings[key]
            except KeyError:
                pass
890
        else:
891
            self.py_strings = {}
892

893 894 895
        if identifier:
            intern = True
        elif identifier is None:
896
            if isinstance(text, bytes):
897
                intern = bool(possible_bytes_identifier(text))
898 899
            else:
                intern = bool(possible_unicode_identifier(text))
900 901 902
        else:
            intern = False
        if intern:
903
            prefix = Naming.interned_prefixes['str']
904 905
        else:
            prefix = Naming.py_const_prefix
906 907 908 909 910 911 912

        if encoding_key:
            encoding_prefix = '_%s' % encoding_key
        else:
            encoding_prefix = ''

        pystring_cname = "%s%s%s_%s" % (
913 914
            prefix,
            (is_str and 's') or (is_unicode and 'u') or 'b',
915
            encoding_prefix,
916 917 918 919 920
            self.cname[len(Naming.const_prefix):])

        py_string = PyStringConst(
            pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
        self.py_strings[key] = py_string
921 922 923 924 925 926
        return py_string

class PyStringConst(object):
    """Global info about a Python string constant held by GlobalState.
    """
    # cname       string
927
    # py3str_cstring string
928
    # encoding    string
929
    # intern      boolean
930 931
    # is_unicode  boolean
    # is_str      boolean
932

933 934
    def __init__(self, cname, encoding, is_unicode, is_str=False,
                 py3str_cstring=None, intern=False):
935
        self.cname = cname
936
        self.py3str_cstring = py3str_cstring
937 938 939
        self.encoding = encoding
        self.is_str = is_str
        self.is_unicode = is_unicode
940 941
        self.intern = intern

Stefan Behnel's avatar
Stefan Behnel committed
942 943 944
    def __lt__(self, other):
        return self.cname < other.cname

945

946 947 948 949 950 951
class GlobalState(object):
    # filename_table   {string : int}  for finding filename table indexes
    # filename_list    [string]        filenames in filename table order
    # input_file_contents dict         contents (=list of lines) of any file that was used as input
    #                                  to create this output C code.  This is
    #                                  used to annotate the comments.
952
    #
953
    # utility_codes   set                IDs of used utility code (to avoid reinsertion)
954 955 956 957 958 959
    #
    # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
    #                                  constants etc. into the pyx-declared ones (i.e,
    #                                  check if constants are already added).
    #                                  In time, hopefully the literals etc. will be
    #                                  supplied directly instead.
960
    #
961
    # const_cnames_used  dict          global counter for unique constant identifiers
962
    #
963

964 965
    # parts            {string:CCodeWriter}

966

967 968 969 970
    # interned_strings
    # consts
    # interned_nums

971 972 973 974 975 976
    # directives       set             Temporary variable used to track
    #                                  the current set of directives in the code generation
    #                                  process.

    directives = {}

977 978
    code_layout = [
        'h_code',
Robert Bradshaw's avatar
Robert Bradshaw committed
979
        'filename_table',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
980
        'utility_code_proto_before_types',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
981 982 983 984
        'numeric_typedefs',          # Let these detailed individual parts stay!,
        'complex_type_declarations', # as the proper solution is to make a full DAG...
        'type_declarations',         # More coarse-grained blocks would simply hide
        'utility_code_proto',        # the ugliness, not fix it
985 986
        'module_declarations',
        'typeinfo',
987 988
        'before_global_var',
        'global_var',
989
        'string_decls',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
990
        'decls',
991
        'all_the_rest',
992 993
        'pystring_table',
        'cached_builtins',
994
        'cached_constants',
995 996 997 998 999
        'init_globals',
        'init_module',
        'cleanup_globals',
        'cleanup_module',
        'main_method',
1000 1001
        'utility_code_def',
        'end'
1002
    ]
1003

1004

1005
    def __init__(self, writer, module_node, code_config, common_utility_include_dir=None):
1006 1007 1008
        self.filename_table = {}
        self.filename_list = []
        self.input_file_contents = {}
Robert Bradshaw's avatar
Robert Bradshaw committed
1009
        self.utility_codes = set()
1010
        self.declared_cnames = {}
1011
        self.in_utility_code_generation = False
1012
        self.code_config = code_config
1013
        self.common_utility_include_dir = common_utility_include_dir
1014
        self.parts = {}
1015 1016
        self.module_node = module_node # because some utility code generation needs it
                                       # (generating backwards-compatible Get/ReleaseBuffer
1017

1018
        self.const_cnames_used = {}
1019
        self.string_const_index = {}
Nikita Nemkin's avatar
Nikita Nemkin committed
1020
        self.pyunicode_ptr_const_index = {}
Stefan Behnel's avatar
Stefan Behnel committed
1021
        self.num_const_index = {}
1022
        self.py_constants = []
1023
        self.cached_cmethods = {}
1024

1025
        writer.set_global_state(self)
1026
        self.rootwriter = writer
1027

1028 1029 1030 1031
    def initialize_main_c_code(self):
        rootwriter = self.rootwriter
        for part in self.code_layout:
            self.parts[part] = rootwriter.insertion_point()
1032

1033 1034 1035 1036 1037 1038
        if not Options.cache_builtins:
            del self.parts['cached_builtins']
        else:
            w = self.parts['cached_builtins']
            w.enter_cfunc_scope()
            w.putln("static int __Pyx_InitCachedBuiltins(void) {")
1039

1040 1041 1042 1043
        w = self.parts['cached_constants']
        w.enter_cfunc_scope()
        w.putln("")
        w.putln("static int __Pyx_InitCachedConstants(void) {")
1044
        w.put_declare_refcount_context()
1045 1046
        w.put_setup_refcount_context("__Pyx_InitCachedConstants")

1047 1048 1049 1050
        w = self.parts['init_globals']
        w.enter_cfunc_scope()
        w.putln("")
        w.putln("static int __Pyx_InitGlobals(void) {")
1051

1052 1053 1054 1055 1056 1057 1058
        if not Options.generate_cleanup_code:
            del self.parts['cleanup_globals']
        else:
            w = self.parts['cleanup_globals']
            w.enter_cfunc_scope()
            w.putln("")
            w.putln("static void __Pyx_CleanupGlobals(void) {")
1059

1060 1061 1062 1063
        code = self.parts['utility_code_proto']
        code.putln("")
        code.putln("/* --- Runtime support code (head) --- */")

1064
        code = self.parts['utility_code_def']
1065
        if self.code_config.emit_linenums:
1066 1067
            code.write('\n#line 1 "cython_utility"\n')
        code.putln("")
1068
        code.putln("/* --- Runtime support code --- */")
1069

1070
    def finalize_main_c_code(self):
1071 1072 1073 1074 1075 1076
        self.close_global_decls()

        #
        # utility_code_def
        #
        code = self.parts['utility_code_def']
1077 1078
        util = TempitaUtilityCode.load_cached("TypeConversions", "TypeConversion.c")
        code.put(util.format_code(util.impl))
1079 1080
        code.putln("")

1081 1082 1083
    def __getitem__(self, key):
        return self.parts[key]

1084 1085 1086
    #
    # Global constants, interned objects, etc.
    #
1087 1088
    def close_global_decls(self):
        # This is called when it is known that no more global declarations will
1089
        # declared.
1090
        self.generate_const_declarations()
1091
        if Options.cache_builtins:
1092
            w = self.parts['cached_builtins']
1093
            w.putln("return 0;")
1094 1095 1096
            if w.label_used(w.error_label):
                w.put_label(w.error_label)
                w.putln("return -1;")
1097 1098 1099
            w.putln("}")
            w.exit_cfunc_scope()

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
        w = self.parts['cached_constants']
        w.put_finish_refcount_context()
        w.putln("return 0;")
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.put_finish_refcount_context()
            w.putln("return -1;")
        w.putln("}")
        w.exit_cfunc_scope()

1110
        w = self.parts['init_globals']
1111
        w.putln("return 0;")
1112 1113 1114
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.putln("return -1;")
1115 1116
        w.putln("}")
        w.exit_cfunc_scope()
1117

1118 1119 1120 1121 1122
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_globals']
            w.putln("}")
            w.exit_cfunc_scope()

1123 1124 1125 1126
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_module']
            w.putln("}")
            w.exit_cfunc_scope()
1127

1128
    def put_pyobject_decl(self, entry):
1129
        self['global_var'].putln("static PyObject *%s;" % entry.cname)
1130

1131 1132
    # constant handling at code generation time

1133 1134 1135
    def get_cached_constants_writer(self):
        return self.parts['cached_constants']

1136
    def get_int_const(self, str_value, longness=False):
Stefan Behnel's avatar
Stefan Behnel committed
1137
        py_type = longness and 'long' or 'int'
1138
        try:
Stefan Behnel's avatar
Stefan Behnel committed
1139
            c = self.num_const_index[(str_value, py_type)]
1140
        except KeyError:
Stefan Behnel's avatar
Stefan Behnel committed
1141 1142 1143 1144 1145 1146 1147 1148
            c = self.new_num_const(str_value, py_type)
        return c

    def get_float_const(self, str_value, value_code):
        try:
            c = self.num_const_index[(str_value, 'float')]
        except KeyError:
            c = self.new_num_const(str_value, 'float', value_code)
1149 1150
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1151
    def get_py_const(self, type, prefix='', cleanup_level=None):
1152
        # create a new Python object constant
Stefan Behnel's avatar
Stefan Behnel committed
1153
        const = self.new_py_const(type, prefix)
1154
        if cleanup_level is not None \
Stefan Behnel's avatar
Stefan Behnel committed
1155
                and cleanup_level <= Options.generate_cleanup_code:
1156
            cleanup_writer = self.parts['cleanup_globals']
1157
            cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
1158
        return const
1159

1160
    def get_string_const(self, text, py_version=None):
1161 1162
        # return a C string constant, creating a new one if necessary
        if text.is_unicode:
1163
            byte_string = text.utf8encode()
1164 1165 1166 1167 1168 1169
        else:
            byte_string = text.byteencode()
        try:
            c = self.string_const_index[byte_string]
        except KeyError:
            c = self.new_string_const(text, byte_string)
1170
        c.add_py_version(py_version)
1171 1172
        return c

1173
    def get_pyunicode_ptr_const(self, text):
1174 1175 1176
        # return a Py_UNICODE[] constant, creating a new one if necessary
        assert text.is_unicode
        try:
Nikita Nemkin's avatar
Nikita Nemkin committed
1177
            c = self.pyunicode_ptr_const_index[text]
1178
        except KeyError:
Nikita Nemkin's avatar
Nikita Nemkin committed
1179
            c = self.pyunicode_ptr_const_index[text] = self.new_const_cname()
1180 1181
        return c

1182 1183
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
1184
        # return a Python string constant, creating a new one if necessary
1185 1186
        py3str_cstring = None
        if is_str and unicode_value is not None \
1187
               and unicode_value.utf8encode() != text.byteencode():
1188 1189 1190 1191
            py3str_cstring = self.get_string_const(unicode_value, py_version=3)
            c_string = self.get_string_const(text, py_version=2)
        else:
            c_string = self.get_string_const(text)
1192 1193
        py_string = c_string.get_py_string_const(
            text.encoding, identifier, is_str, py3str_cstring)
1194 1195
        return py_string

1196 1197 1198
    def get_interned_identifier(self, text):
        return self.get_py_string_const(text, identifier=True)

1199
    def new_string_const(self, text, byte_string):
Stefan Behnel's avatar
Stefan Behnel committed
1200
        cname = self.new_string_const_cname(byte_string)
1201 1202 1203 1204
        c = StringConst(cname, text, byte_string)
        self.string_const_index[byte_string] = c
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1205 1206 1207 1208
    def new_num_const(self, value, py_type, value_code=None):
        cname = self.new_num_const_cname(value, py_type)
        c = NumConst(cname, value, py_type, value_code)
        self.num_const_index[(value, py_type)] = c
1209 1210
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1211 1212
    def new_py_const(self, type, prefix=''):
        cname = self.new_const_cname(prefix)
1213 1214 1215 1216
        c = PyObjectConst(cname, type)
        self.py_constants.append(c)
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1217
    def new_string_const_cname(self, bytes_value):
1218
        # Create a new globally-unique nice name for a C string constant.
Stefan Behnel's avatar
Stefan Behnel committed
1219
        value = bytes_value.decode('ASCII', 'ignore')
1220
        return self.new_const_cname(value=value)
1221

Stefan Behnel's avatar
Stefan Behnel committed
1222 1223
    def new_num_const_cname(self, value, py_type):
        if py_type == 'long':
1224
            value += 'L'
1225 1226
            py_type = 'int'
        prefix = Naming.interned_prefixes[py_type]
Stefan Behnel's avatar
Stefan Behnel committed
1227
        cname = "%s%s" % (prefix, value)
1228
        cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_')
1229 1230
        return cname

1231 1232
    def new_const_cname(self, prefix='', value=''):
        value = replace_identifier('_', value)[:32].strip('_')
1233 1234 1235
        used = self.const_cnames_used
        name_suffix = value
        while name_suffix in used:
1236
            counter = used[value] = used[value] + 1
1237
            name_suffix = '%s_%d' % (value, counter)
1238
        used[name_suffix] = 1
1239 1240 1241 1242 1243
        if prefix:
            prefix = Naming.interned_prefixes[prefix]
        else:
            prefix = Naming.const_prefix
        return "%s%s" % (prefix, name_suffix)
1244

1245 1246
    def get_cached_unbound_method(self, type_cname, method_name, args_count):
        key = (type_cname, method_name, args_count)
1247 1248 1249 1250 1251 1252 1253
        try:
            cname = self.cached_cmethods[key]
        except KeyError:
            cname = self.cached_cmethods[key] = self.new_const_cname(
                'umethod', '%s_%s' % (type_cname, method_name))
        return cname

1254
    def add_cached_builtin_decl(self, entry):
1255
        if entry.is_builtin and entry.is_const:
1256 1257
            if self.should_declare(entry.cname, entry):
                self.put_pyobject_decl(entry)
1258
                w = self.parts['cached_builtins']
1259 1260 1261 1262
                condition = None
                if entry.name in non_portable_builtins_map:
                    condition, replacement = non_portable_builtins_map[entry.name]
                    w.putln('#if %s' % condition)
1263
                    self.put_cached_builtin_init(
1264
                        entry.pos, StringEncoding.EncodedString(replacement),
1265
                        entry.cname)
1266 1267 1268 1269
                    w.putln('#else')
                self.put_cached_builtin_init(
                    entry.pos, StringEncoding.EncodedString(entry.name),
                    entry.cname)
1270
                if condition:
1271 1272 1273 1274 1275
                    w.putln('#endif')

    def put_cached_builtin_init(self, pos, name, cname):
        w = self.parts['cached_builtins']
        interned_cname = self.get_interned_identifier(name).cname
1276 1277 1278
        self.use_utility_code(
            UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c"))
        w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % (
1279 1280 1281 1282
            cname,
            interned_cname,
            cname,
            w.error_goto(pos)))
1283 1284

    def generate_const_declarations(self):
1285
        self.generate_cached_methods_decls()
1286
        self.generate_string_constants()
Stefan Behnel's avatar
Stefan Behnel committed
1287
        self.generate_num_constants()
1288 1289 1290
        self.generate_object_constant_decls()

    def generate_object_constant_decls(self):
Stefan Behnel's avatar
Stefan Behnel committed
1291 1292
        consts = [(len(c.cname), c.cname, c)
                  for c in self.py_constants]
1293
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1294
        decls_writer = self.parts['decls']
1295
        for _, cname, c in consts:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1296
            decls_writer.putln(
1297
                "static %s;" % c.type.declaration_code(cname))
1298

1299 1300 1301 1302
    def generate_cached_methods_decls(self):
        if not self.cached_cmethods:
            return

1303 1304
        decl = self.parts['decls']
        init = self.parts['init_globals']
1305
        cnames = []
1306
        for (type_cname, method_name, _), cname in sorted(self.cached_cmethods.items()):
1307 1308
            cnames.append(cname)
            method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname
1309 1310 1311 1312 1313
            decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % (
                cname, method_name_cname))
            # split type reference storage as it might not be static
            init.putln('%s.type = (PyObject*)&%s;' % (
                cname, type_cname))
1314 1315 1316 1317 1318 1319

        if Options.generate_cleanup_code:
            cleanup = self.parts['cleanup_globals']
            for cname in cnames:
                cleanup.putln("Py_CLEAR(%s.method);" % cname)

1320
    def generate_string_constants(self):
1321
        c_consts = [(len(c.cname), c.cname, c) for c in self.string_const_index.values()]
1322 1323
        c_consts.sort()
        py_strings = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1324

1325
        decls_writer = self.parts['string_decls']
1326
        for _, cname, c in c_consts:
1327 1328 1329 1330 1331
            conditional = False
            if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
                conditional = True
                decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
                    (2 in c.py_versions) and '<' or '>='))
1332
            decls_writer.putln('static const char %s[] = "%s";' % (
1333
                cname, StringEncoding.split_string_literal(c.escaped_value)))
1334 1335
            if conditional:
                decls_writer.putln("#endif")
1336
            if c.py_strings is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1337
                for py_string in c.py_strings.values():
1338 1339
                    py_strings.append((c.cname, len(py_string.cname), py_string))

1340
        for c, cname in sorted(self.pyunicode_ptr_const_index.items()):
1341
            utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
1342 1343
            if utf16_array:
                # Narrow and wide representations differ
Nikita Nemkin's avatar
Nikita Nemkin committed
1344
                decls_writer.putln("#ifdef Py_UNICODE_WIDE")
1345 1346 1347 1348 1349
            decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array))
            if utf16_array:
                decls_writer.putln("#else")
                decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
                decls_writer.putln("#endif")
1350

1351
        if py_strings:
1352
            self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
1353
            py_strings.sort()
1354 1355
            w = self.parts['pystring_table']
            w.putln("")
Stefan Behnel's avatar
Stefan Behnel committed
1356
            w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname)
1357
            for c_cname, _, py_string in py_strings:
1358
                if not py_string.is_str or not py_string.encoding or \
Stefan Behnel's avatar
Stefan Behnel committed
1359 1360
                        py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
                                               'UTF8', 'UTF-8'):
1361 1362 1363 1364
                    encoding = '0'
                else:
                    encoding = '"%s"' % py_string.encoding.lower()

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1365
                decls_writer.putln(
1366
                    "static PyObject *%s;" % py_string.cname)
1367 1368
                if py_string.py3str_cstring:
                    w.putln("#if PY_MAJOR_VERSION >= 3")
Stefan Behnel's avatar
Stefan Behnel committed
1369
                    w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1370 1371 1372
                        py_string.cname,
                        py_string.py3str_cstring.cname,
                        py_string.py3str_cstring.cname,
1373 1374
                        '0', 1, 0,
                        py_string.intern
1375 1376
                        ))
                    w.putln("#else")
Stefan Behnel's avatar
Stefan Behnel committed
1377
                w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1378 1379 1380
                    py_string.cname,
                    c_cname,
                    c_cname,
1381 1382 1383 1384
                    encoding,
                    py_string.is_unicode,
                    py_string.is_str,
                    py_string.intern
1385
                    ))
1386 1387
                if py_string.py3str_cstring:
                    w.putln("#endif")
1388
            w.putln("{0, 0, 0, 0, 0, 0, 0}")
1389
            w.putln("};")
1390

1391 1392
            init_globals = self.parts['init_globals']
            init_globals.putln(
1393 1394
                "if (__Pyx_InitStrings(%s) < 0) %s;" % (
                    Naming.stringtab_cname,
1395
                    init_globals.error_goto(self.module_pos)))
1396

Stefan Behnel's avatar
Stefan Behnel committed
1397
    def generate_num_constants(self):
1398
        consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
Stefan Behnel's avatar
Stefan Behnel committed
1399
                  for c in self.num_const_index.values()]
1400
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1401
        decls_writer = self.parts['decls']
Stefan Behnel's avatar
Stefan Behnel committed
1402
        init_globals = self.parts['init_globals']
1403
        for py_type, _, _, value, value_code, c in consts:
1404
            cname = c.cname
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1405
            decls_writer.putln("static PyObject *%s;" % cname)
Stefan Behnel's avatar
Stefan Behnel committed
1406
            if py_type == 'float':
Stefan Behnel's avatar
Stefan Behnel committed
1407
                function = 'PyFloat_FromDouble(%s)'
Stefan Behnel's avatar
Stefan Behnel committed
1408
            elif py_type == 'long':
Stefan Behnel's avatar
Stefan Behnel committed
1409
                function = 'PyLong_FromString((char *)"%s", 0, 0)'
1410
            elif Utils.long_literal(value):
Stefan Behnel's avatar
Stefan Behnel committed
1411
                function = 'PyInt_FromString((char *)"%s", 0, 0)'
1412 1413
            elif len(value.lstrip('-')) > 4:
                function = "PyInt_FromLong(%sL)"
1414
            else:
Stefan Behnel's avatar
Stefan Behnel committed
1415 1416 1417
                function = "PyInt_FromLong(%s)"
            init_globals.putln('%s = %s; %s' % (
                cname, function % value_code,
1418
                init_globals.error_goto_if_null(cname, self.module_pos)))
1419

1420 1421 1422
    # The functions below are there in a transition phase only
    # and will be deprecated. They are called from Nodes.BlockNode.
    # The copy&paste duplication is intentional in order to be able
1423
    # to see quickly how BlockNode worked, until this is replaced.
1424

1425 1426 1427
    def should_declare(self, cname, entry):
        if cname in self.declared_cnames:
            other = self.declared_cnames[cname]
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1428
            assert str(entry.type) == str(other.type)
1429 1430 1431 1432 1433 1434 1435 1436 1437
            assert entry.init == other.init
            return False
        else:
            self.declared_cnames[cname] = entry
            return True

    #
    # File name state
    #
1438

1439
    def lookup_filename(self, source_desc):
1440
        entry = source_desc.get_filenametable_entry()
1441
        try:
1442
            index = self.filename_table[entry]
1443 1444
        except KeyError:
            index = len(self.filename_list)
1445
            self.filename_list.append(source_desc)
1446
            self.filename_table[entry] = index
1447 1448 1449 1450 1451 1452
        return index

    def commented_file_contents(self, source_desc):
        try:
            return self.input_file_contents[source_desc]
        except KeyError:
1453 1454 1455 1456
            pass
        source_file = source_desc.get_lines(encoding='ASCII',
                                            error_handling='ignore')
        try:
1457 1458
            F = [u' * ' + line.rstrip().replace(
                    u'*/', u'*[inserted by cython to avoid comment closer]/'
1459 1460
                    ).replace(
                    u'/*', u'/[inserted by cython to avoid comment start]*'
1461
                    )
1462 1463 1464 1465 1466 1467 1468
                 for line in source_file]
        finally:
            if hasattr(source_file, 'close'):
                source_file.close()
        if not F: F.append(u'')
        self.input_file_contents[source_desc] = F
        return F
1469

1470 1471 1472
    #
    # Utility code state
    #
1473

1474
    def use_utility_code(self, utility_code):
1475
        """
1476 1477 1478 1479
        Adds code to the C file. utility_code should
        a) implement __eq__/__hash__ for the purpose of knowing whether the same
           code has already been included
        b) implement put_code, which takes a globalstate instance
1480

1481
        See UtilityCode.
1482
        """
1483
        if utility_code and utility_code not in self.utility_codes:
1484 1485
            self.utility_codes.add(utility_code)
            utility_code.put_code(self)
1486

1487 1488 1489 1490 1491 1492 1493 1494
    def use_entry_utility_code(self, entry):
        if entry is None:
            return
        if entry.utility_code:
            self.use_utility_code(entry.utility_code)
        if entry.utility_code_definition:
            self.use_utility_code(entry.utility_code_definition)

1495

1496
def funccontext_property(name):
1497
    attribute_of = operator.attrgetter(name)
1498
    def get(self):
1499
        return attribute_of(self.funcstate)
1500
    def set(self, value):
1501
        setattr(self.funcstate, name, value)
1502 1503
    return property(get, set)

1504

1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
class CCodeConfig(object):
    # emit_linenums       boolean         write #line pragmas?
    # emit_code_comments  boolean         copy the original code into C comments?
    # c_line_in_traceback boolean         append the c file and line number to the traceback for exceptions?

    def __init__(self, emit_linenums=True, emit_code_comments=True, c_line_in_traceback=True):
        self.emit_code_comments = emit_code_comments
        self.emit_linenums = emit_linenums
        self.c_line_in_traceback = c_line_in_traceback


1516
class CCodeWriter(object):
1517
    """
1518
    Utility class to output C code.
1519

1520
    When creating an insertion point one must care about the state that is
1521
    kept:
1522 1523
    - formatting state (level, bol) is cloned and used in insertion points
      as well
1524 1525
    - labels, temps, exc_vars: One must construct a scope in which these can
      exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
1526 1527 1528 1529 1530
      sanity checking and forward compatabilty). Created insertion points
      looses this scope and cannot access it.
    - marker: Not copied to insertion point
    - filename_table, filename_list, input_file_contents: All codewriters
      coming from the same root share the same instances simultaneously.
1531
    """
1532

1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
    # f                   file            output file
    # buffer              StringIOTree

    # level               int             indentation level
    # bol                 bool            beginning of line?
    # marker              string          comment to emit before next line
    # funcstate           FunctionState   contains state local to a C function used for code
    #                                     generation (labels and temps state etc.)
    # globalstate         GlobalState     contains state global for a C file (input file info,
    #                                     utility code, declared constants etc.)
    # pyclass_stack       list            used during recursive code generation to pass information
    #                                     about the current class one is in
1545
    # code_config         CCodeConfig     configuration options for the C code writer
1546

1547
    globalstate = code_config = None
1548

1549
    def __init__(self, create_from=None, buffer=None, copy_formatting=False):
1550 1551
        if buffer is None: buffer = StringIOTree()
        self.buffer = buffer
1552 1553
        self.last_pos = None
        self.last_marked_pos = None
1554
        self.pyclass_stack = []
1555

1556 1557
        self.funcstate = None
        self.level = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
1558
        self.call_level = 0
1559
        self.bol = 1
1560 1561

        if create_from is not None:
1562
            # Use same global state
1563
            self.set_global_state(create_from.globalstate)
1564
            self.funcstate = create_from.funcstate
1565
            # Clone formatting state
1566 1567 1568
            if copy_formatting:
                self.level = create_from.level
                self.bol = create_from.bol
Robert Bradshaw's avatar
Robert Bradshaw committed
1569
                self.call_level = create_from.call_level
1570 1571
            self.last_pos = create_from.last_pos
            self.last_marked_pos = create_from.last_marked_pos
1572

1573
    def create_new(self, create_from, buffer, copy_formatting):
1574 1575
        # polymorphic constructor -- very slightly more versatile
        # than using __class__
1576
        result = CCodeWriter(create_from, buffer, copy_formatting)
1577
        return result
1578

1579 1580 1581 1582 1583
    def set_global_state(self, global_state):
        assert self.globalstate is None  # prevent overwriting once it's set
        self.globalstate = global_state
        self.code_config = global_state.code_config

1584 1585 1586
    def copyto(self, f):
        self.buffer.copyto(f)

1587 1588 1589 1590
    def getvalue(self):
        return self.buffer.getvalue()

    def write(self, s):
1591
        # also put invalid markers (lineno 0), to indicate that those lines
Mark Florisson's avatar
Mark Florisson committed
1592
        # have no Cython source code correspondence
1593
        cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0
Mark Florisson's avatar
Mark Florisson committed
1594
        self.buffer.markers.extend([cython_lineno] * s.count('\n'))
1595
        self.buffer.write(s)
1596 1597

    def insertion_point(self):
1598
        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
1599 1600
        return other

1601 1602 1603 1604 1605
    def new_writer(self):
        """
        Creates a new CCodeWriter connected to the same global state, which
        can later be inserted using insert.
        """
1606
        return CCodeWriter(create_from=self)
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616

    def insert(self, writer):
        """
        Inserts the contents of another code writer (created with
        the same global state) in the current location.

        It is ok to write to the inserted writer also after insertion.
        """
        assert writer.globalstate is self.globalstate
        self.buffer.insert(writer.buffer)
1617 1618 1619 1620 1621 1622 1623 1624

    # Properties delegated to function scope
    label_counter = funccontext_property("label_counter")
    return_label = funccontext_property("return_label")
    error_label = funccontext_property("error_label")
    labels_used = funccontext_property("labels_used")
    continue_label = funccontext_property("continue_label")
    break_label = funccontext_property("break_label")
1625
    return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label")
1626
    yield_labels = funccontext_property("yield_labels")
1627 1628

    # Functions delegated to function scope
Dag Sverre Seljebotn's avatar
merge  
Dag Sverre Seljebotn committed
1629
    def new_label(self, name=None):    return self.funcstate.new_label(name)
1630
    def new_error_label(self):         return self.funcstate.new_error_label()
1631
    def new_yield_label(self):         return self.funcstate.new_yield_label()
1632 1633 1634 1635 1636 1637 1638 1639
    def get_loop_labels(self):         return self.funcstate.get_loop_labels()
    def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
    def new_loop_labels(self):         return self.funcstate.new_loop_labels()
    def get_all_labels(self):          return self.funcstate.get_all_labels()
    def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
    def all_new_labels(self):          return self.funcstate.all_new_labels()
    def use_label(self, lbl):          return self.funcstate.use_label(lbl)
    def label_used(self, lbl):         return self.funcstate.label_used(lbl)
1640 1641


1642 1643
    def enter_cfunc_scope(self, scope=None):
        self.funcstate = FunctionState(self, scope=scope)
1644

1645
    def exit_cfunc_scope(self):
1646
        self.funcstate = None
1647

1648 1649
    # constant handling

Stefan Behnel's avatar
Stefan Behnel committed
1650
    def get_py_int(self, str_value, longness):
1651 1652
        return self.globalstate.get_int_const(str_value, longness).cname

Stefan Behnel's avatar
Stefan Behnel committed
1653 1654 1655
    def get_py_float(self, str_value, value_code):
        return self.globalstate.get_float_const(str_value, value_code).cname

1656 1657
    def get_py_const(self, type, prefix='', cleanup_level=None):
        return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
1658

1659 1660 1661
    def get_string_const(self, text):
        return self.globalstate.get_string_const(text).cname

1662 1663
    def get_pyunicode_ptr_const(self, text):
        return self.globalstate.get_pyunicode_ptr_const(text)
1664

1665 1666 1667 1668
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
        return self.globalstate.get_py_string_const(
            text, identifier, is_str, unicode_value).cname
1669

1670 1671 1672
    def get_argument_default_const(self, type):
        return self.globalstate.get_py_const(type).cname

1673 1674 1675 1676
    def intern(self, text):
        return self.get_py_string_const(text)

    def intern_identifier(self, text):
1677
        return self.get_py_string_const(text, identifier=True)
1678

1679 1680 1681
    def get_cached_constants_writer(self):
        return self.globalstate.get_cached_constants_writer()

1682 1683
    # code generation

Stefan Behnel's avatar
Stefan Behnel committed
1684
    def putln(self, code="", safe=False):
1685
        if self.last_pos and self.bol:
William Stein's avatar
William Stein committed
1686
            self.emit_marker()
1687
        if self.code_config.emit_linenums and self.last_marked_pos:
1688 1689
            source_desc, line, _ = self.last_marked_pos
            self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description()))
William Stein's avatar
William Stein committed
1690
        if code:
1691 1692 1693 1694
            if safe:
                self.put_safe(code)
            else:
                self.put(code)
1695
        self.write("\n")
William Stein's avatar
William Stein committed
1696
        self.bol = 1
1697

1698
    def mark_pos(self, pos, trace=True):
1699 1700 1701 1702
        if pos is None:
            return
        if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]:
            return
1703
        self.last_pos = (pos, trace)
1704

William Stein's avatar
William Stein committed
1705
    def emit_marker(self):
1706 1707
        pos, trace = self.last_pos
        self.last_marked_pos = pos
1708
        self.last_pos = None
1709
        self.write("\n")
1710
        if self.code_config.emit_code_comments:
1711 1712
            self.indent()
            self.write("/* %s */\n" % self._build_marker(pos))
1713
        if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']:
1714
            self.indent()
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
            self.write('__Pyx_TraceLine(%d,%d,%s)\n' % (
                pos[1], not self.funcstate.gil_owned, self.error_goto(pos)))

    def _build_marker(self, pos):
        source_desc, line, col = pos
        assert isinstance(source_desc, SourceDescriptor)
        contents = self.globalstate.commented_file_contents(source_desc)
        lines = contents[max(0, line-3):line]  # line numbers start at 1
        lines[-1] += u'             # <<<<<<<<<<<<<<'
        lines += contents[line:line+2]
        return u'"%s":%d\n%s\n' % (source_desc.get_escaped_description(), line, u'\n'.join(lines))
William Stein's avatar
William Stein committed
1726

1727 1728 1729 1730 1731
    def put_safe(self, code):
        # put code, but ignore {}
        self.write(code)
        self.bol = 0

1732
    def put_or_include(self, code, name):
Stefan Behnel's avatar
Stefan Behnel committed
1733
        include_dir = self.globalstate.common_utility_include_dir
1734
        if include_dir and len(code) > 1024:
Stefan Behnel's avatar
Stefan Behnel committed
1735
            include_file = "%s_%s.h" % (
1736
                name, hashlib.md5(code.encode('utf8')).hexdigest())
Stefan Behnel's avatar
Stefan Behnel committed
1737 1738 1739
            path = os.path.join(include_dir, include_file)
            if not os.path.exists(path):
                tmp_path = '%s.tmp%s' % (path, os.getpid())
Stefan Behnel's avatar
Stefan Behnel committed
1740
                with closing(Utils.open_new_file(tmp_path)) as f:
Stefan Behnel's avatar
Stefan Behnel committed
1741
                    f.write(code)
1742
                shutil.move(tmp_path, path)
Stefan Behnel's avatar
Stefan Behnel committed
1743 1744
            code = '#include "%s"\n' % path
        self.put(code)
1745

William Stein's avatar
William Stein committed
1746
    def put(self, code):
1747 1748
        if is_self_assignment(code):
            return
1749
        fix_indent = False
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
        if "{" in code:
            dl = code.count("{")
        else:
            dl = 0
        if "}" in code:
            dl -= code.count("}")
            if dl < 0:
                self.level += dl
            elif dl == 0 and code[0] == "}":
                # special cases like "} else {" need a temporary dedent
                fix_indent = True
                self.level -= 1
William Stein's avatar
William Stein committed
1762 1763
        if self.bol:
            self.indent()
1764
        self.write(code)
William Stein's avatar
William Stein committed
1765 1766 1767
        self.bol = 0
        if dl > 0:
            self.level += dl
1768
        elif fix_indent:
1769 1770
            self.level += 1

Mark Florisson's avatar
Mark Florisson committed
1771
    def putln_tempita(self, code, **context):
1772
        from ..Tempita import sub
1773
        self.putln(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1774 1775

    def put_tempita(self, code, **context):
1776
        from ..Tempita import sub
1777
        self.put(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1778

William Stein's avatar
William Stein committed
1779
    def increase_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1780
        self.level += 1
1781

William Stein's avatar
William Stein committed
1782
    def decrease_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1783
        self.level -= 1
1784

William Stein's avatar
William Stein committed
1785 1786 1787
    def begin_block(self):
        self.putln("{")
        self.increase_indent()
1788

William Stein's avatar
William Stein committed
1789 1790 1791
    def end_block(self):
        self.decrease_indent()
        self.putln("}")
1792

William Stein's avatar
William Stein committed
1793
    def indent(self):
1794
        self.write("  " * self.level)
1795

1796 1797 1798
    def get_py_version_hex(self, pyversion):
        return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]

William Stein's avatar
William Stein committed
1799
    def put_label(self, lbl):
1800
        if lbl in self.funcstate.labels_used:
1801
            self.putln("%s:;" % lbl)
1802

1803
    def put_goto(self, lbl):
1804
        self.funcstate.use_label(lbl)
1805
        self.putln("goto %s;" % lbl)
1806

1807
    def put_var_declaration(self, entry, storage_class="",
Stefan Behnel's avatar
Stefan Behnel committed
1808
                            dll_linkage=None, definition=True):
1809
        #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
1810 1811
        if entry.visibility == 'private' and not (definition or entry.defined_in_pxd):
            #print "...private and not definition, skipping", entry.cname ###
1812
            return
1813 1814
        if entry.visibility == "private" and not entry.used:
            #print "...private and not used, skipping", entry.cname ###
William Stein's avatar
William Stein committed
1815 1816 1817
            return
        if storage_class:
            self.put("%s " % storage_class)
1818 1819
        if not entry.cf_used:
            self.put('CYTHON_UNUSED ')
1820
        self.put(entry.type.declaration_code(
Stefan Behnel's avatar
Stefan Behnel committed
1821
            entry.cname, dll_linkage=dll_linkage))
William Stein's avatar
William Stein committed
1822
        if entry.init is not None:
1823
            self.put_safe(" = %s" % entry.type.literal_code(entry.init))
1824
        elif entry.type.is_pyobject:
Stefan Behnel's avatar
Stefan Behnel committed
1825
            self.put(" = NULL")
William Stein's avatar
William Stein committed
1826
        self.putln(";")
1827 1828

    def put_temp_declarations(self, func_context):
1829
        for name, type, manage_ref, static in func_context.temps_allocated:
1830 1831 1832
            decl = type.declaration_code(name)
            if type.is_pyobject:
                self.putln("%s = NULL;" % decl)
1833
            elif type.is_memoryviewslice:
1834
                from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
1835
                self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init))
1836
            else:
1837
                self.putln("%s%s;" % (static and "static " or "", decl))
1838

1839
        if func_context.should_declare_error_indicator:
1840 1841 1842 1843
            if self.funcstate.uses_error_indicator:
                unused = ''
            else:
                unused = 'CYTHON_UNUSED '
1844
            # Initialize these variables to silence compiler warnings
1845 1846 1847
            self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname))
            self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname))
            self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))
1848

1849 1850 1851 1852
    def put_generated_by(self):
        self.putln("/* Generated by Cython %s */" % Version.watermark)
        self.putln("")

1853 1854 1855
    def put_h_guard(self, guard):
        self.putln("#ifndef %s" % guard)
        self.putln("#define %s" % guard)
1856

1857 1858 1859 1860 1861 1862
    def unlikely(self, cond):
        if Options.gcc_branch_hints:
            return 'unlikely(%s)' % cond
        else:
            return cond

1863 1864 1865 1866 1867
    def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper):
        if not modifiers:
            return ''
        return '%s ' % ' '.join([mapper(m,m) for m in modifiers])

1868 1869
    # Python objects and reference counting

William Stein's avatar
William Stein committed
1870 1871
    def entry_as_pyobject(self, entry):
        type = entry.type
1872 1873
        if (not entry.is_self_arg and not entry.type.is_complete()
            or entry.type.is_extension_type):
William Stein's avatar
William Stein committed
1874 1875 1876
            return "(PyObject *)" + entry.cname
        else:
            return entry.cname
1877

William Stein's avatar
William Stein committed
1878
    def as_pyobject(self, cname, type):
1879
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
1880
        return typecast(py_object_type, type, cname)
1881

1882
    def put_gotref(self, cname):
1883
        self.putln("__Pyx_GOTREF(%s);" % cname)
1884

1885 1886
    def put_giveref(self, cname):
        self.putln("__Pyx_GIVEREF(%s);" % cname)
1887

1888 1889 1890
    def put_xgiveref(self, cname):
        self.putln("__Pyx_XGIVEREF(%s);" % cname)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1891 1892 1893
    def put_xgotref(self, cname):
        self.putln("__Pyx_XGOTREF(%s);" % cname)

1894 1895 1896 1897 1898
    def put_incref(self, cname, type, nanny=True):
        if nanny:
            self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
        else:
            self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
1899

1900
    def put_decref(self, cname, type, nanny=True):
1901
        self._put_decref(cname, type, nanny, null_check=False, clear=False)
1902 1903 1904 1905

    def put_var_gotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
1906

1907 1908 1909 1910
    def put_var_giveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))

1911 1912 1913 1914
    def put_var_xgotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))

1915 1916 1917 1918
    def put_var_xgiveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))

William Stein's avatar
William Stein committed
1919 1920
    def put_var_incref(self, entry):
        if entry.type.is_pyobject:
1921
            self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
1922

1923 1924 1925 1926
    def put_var_xincref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry))

1927 1928 1929
    def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=False,
                         clear=True, clear_before_decref=clear_before_decref)
1930

1931
    def put_xdecref(self, cname, type, nanny=True, have_gil=True):
1932 1933
        self._put_decref(cname, type, nanny, null_check=True,
                         have_gil=have_gil, clear=False)
1934

1935 1936 1937
    def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=True,
                         clear=True, clear_before_decref=clear_before_decref)
1938

1939 1940
    def _put_decref(self, cname, type, nanny=True, null_check=False,
                    have_gil=True, clear=False, clear_before_decref=False):
1941
        if type.is_memoryviewslice:
1942
            self.put_xdecref_memoryviewslice(cname, have_gil=have_gil)
1943 1944
            return

1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955
        prefix = nanny and '__Pyx' or 'Py'
        X = null_check and 'X' or ''

        if clear:
            if clear_before_decref:
                if not nanny:
                    X = ''  # CPython doesn't have a Py_XCLEAR()
                self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname))
            else:
                self.putln("%s_%sDECREF(%s); %s = 0;" % (
                    prefix, X, self.as_pyobject(cname, type), cname))
1956
        else:
1957 1958
            self.putln("%s_%sDECREF(%s);" % (
                prefix, X, self.as_pyobject(cname, type)))
William Stein's avatar
William Stein committed
1959

1960 1961 1962 1963 1964 1965
    def put_decref_set(self, cname, rhs_cname):
        self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname))

    def put_xdecref_set(self, cname, rhs_cname):
        self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname))

William Stein's avatar
William Stein committed
1966 1967
    def put_var_decref(self, entry):
        if entry.type.is_pyobject:
1968
            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1969

William Stein's avatar
William Stein committed
1970 1971
    def put_var_xdecref(self, entry):
        if entry.type.is_pyobject:
1972
            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1973

1974 1975 1976
    def put_var_decref_clear(self, entry):
        self._put_var_decref_clear(entry, null_check=False)

William Stein's avatar
William Stein committed
1977
    def put_var_xdecref_clear(self, entry):
1978 1979 1980
        self._put_var_decref_clear(entry, null_check=True)

    def _put_var_decref_clear(self, entry, null_check):
William Stein's avatar
William Stein committed
1981
        if entry.type.is_pyobject:
1982 1983 1984 1985 1986 1987 1988 1989 1990
            if entry.in_closure:
                # reset before DECREF to make sure closure state is
                # consistent during call to DECREF()
                self.putln("__Pyx_%sCLEAR(%s);" % (
                    null_check and 'X' or '',
                    entry.cname))
            else:
                self.putln("__Pyx_%sDECREF(%s); %s = 0;" % (
                    null_check and 'X' or '',
1991
                    self.entry_as_pyobject(entry),
1992
                    entry.cname))
1993

1994
    def put_var_decrefs(self, entries, used_only = 0):
William Stein's avatar
William Stein committed
1995
        for entry in entries:
1996 1997 1998 1999 2000
            if not used_only or entry.used:
                if entry.xdecref_cleanup:
                    self.put_var_xdecref(entry)
                else:
                    self.put_var_decref(entry)
2001

William Stein's avatar
William Stein committed
2002 2003 2004
    def put_var_xdecrefs(self, entries):
        for entry in entries:
            self.put_var_xdecref(entry)
2005

William Stein's avatar
William Stein committed
2006 2007 2008
    def put_var_xdecrefs_clear(self, entries):
        for entry in entries:
            self.put_var_xdecref_clear(entry)
2009

2010
    def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
2011
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
2012
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
2013 2014 2015
        self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
2016
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
2017
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
2018 2019 2020 2021 2022
        self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xgiveref_memoryviewslice(self, slice_cname):
        self.put_xgiveref("%s.memview" % slice_cname)

2023
    def put_init_to_py_none(self, cname, type, nanny=True):
2024
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
2025
        py_none = typecast(type, py_object_type, "Py_None")
2026 2027 2028 2029
        if nanny:
            self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
        else:
            self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
2030

2031
    def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
William Stein's avatar
William Stein committed
2032 2033
        code = template % entry.cname
        #if entry.type.is_extension_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
2034
        #    code = "((PyObject*)%s)" % code
2035
        self.put_init_to_py_none(code, entry.type, nanny)
2036
        if entry.in_closure:
Stefan Behnel's avatar
Stefan Behnel committed
2037
            self.put_giveref('Py_None')
William Stein's avatar
William Stein committed
2038

2039
    def put_pymethoddef(self, entry, term, allow_skip=True):
2040
        if entry.is_special or entry.name == '__getattribute__':
2041
            if entry.name not in special_py_methods:
2042 2043
                if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
                    pass
2044 2045 2046
                # Python's typeobject.c will automatically fill in our slot
                # in add_operators() (called by PyType_Ready) with a value
                # that's better than ours.
2047
                elif allow_skip:
2048
                    return
2049
        from .TypeSlots import method_coexist
William Stein's avatar
William Stein committed
2050 2051 2052 2053
        if entry.doc:
            doc_code = entry.doc_cname
        else:
            doc_code = 0
2054 2055
        method_flags = entry.signature.method_flags()
        if method_flags:
2056
            if entry.is_special:
2057
                method_flags += [method_coexist]
2058
            self.putln(
2059
                '{"%s", (PyCFunction)%s, %s, %s}%s' % (
2060
                    entry.name,
2061
                    entry.func_cname,
2062
                    "|".join(method_flags),
2063 2064
                    doc_code,
                    term))
2065

2066 2067
    # GIL methods

2068
    def put_ensure_gil(self, declare_gilstate=True, variable=None):
2069 2070 2071 2072 2073 2074
        """
        Acquire the GIL. The generated code is safe even when no PyThreadState
        has been allocated for this thread (for threads not initialized by
        using the Python API). Additionally, the code generated by this method
        may be called recursively.
        """
2075 2076
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2077
        self.putln("#ifdef WITH_THREAD")
2078 2079 2080 2081
        if not variable:
            variable = '__pyx_gilstate_save'
            if declare_gilstate:
                self.put("PyGILState_STATE ")
2082
        self.putln("%s = __Pyx_PyGILState_Ensure();" % variable)
2083 2084
        self.putln("#endif")

2085
    def put_release_ensured_gil(self, variable=None):
2086 2087 2088
        """
        Releases the GIL, corresponds to `put_ensure_gil`.
        """
2089 2090
        if not variable:
            variable = '__pyx_gilstate_save'
2091
        self.putln("#ifdef WITH_THREAD")
2092
        self.putln("__Pyx_PyGILState_Release(%s);" % variable)
2093 2094
        self.putln("#endif")

2095
    def put_acquire_gil(self, variable=None):
2096 2097 2098 2099
        """
        Acquire the GIL. The thread's thread state must have been initialized
        by a previous `put_release_gil`
        """
2100 2101 2102
        self.putln("#ifdef WITH_THREAD")
        if variable:
            self.putln('_save = %s;' % variable)
2103
        self.putln("Py_BLOCK_THREADS")
2104
        self.putln("#endif")
2105

2106
    def put_release_gil(self, variable=None):
2107 2108
        "Release the GIL, corresponds to `put_acquire_gil`."
        self.putln("#ifdef WITH_THREAD")
2109
        self.putln("PyThreadState *_save;")
2110
        self.putln("Py_UNBLOCK_THREADS")
2111 2112 2113
        if variable:
            self.putln('%s = _save;' % variable)
        self.putln("#endif")
2114

2115 2116
    def declare_gilstate(self):
        self.putln("#ifdef WITH_THREAD")
2117
        self.putln("PyGILState_STATE __pyx_gilstate_save;")
2118 2119
        self.putln("#endif")

2120 2121
    # error handling

Robert Bradshaw's avatar
Robert Bradshaw committed
2122 2123 2124 2125
    def put_error_if_neg(self, pos, value):
#        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
        return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))

2126
    def put_error_if_unbound(self, pos, entry, in_nogil_context=False):
2127
        from . import ExprNodes
2128
        if entry.from_closure:
2129 2130 2131
            func = '__Pyx_RaiseClosureNameError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_closure_name_error_utility_code)
2132 2133 2134 2135
        elif entry.type.is_memoryviewslice and in_nogil_context:
            func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_memoryview_utility_code_nogil)
2136
        else:
2137 2138 2139
            func = '__Pyx_RaiseUnboundLocalError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_local_error_utility_code)
Mark Florisson's avatar
Mark Florisson committed
2140

2141
        self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (
Mark Florisson's avatar
Mark Florisson committed
2142 2143 2144 2145
                                entry.type.check_for_null_code(entry.cname),
                                func,
                                entry.name,
                                self.error_goto(pos)))
2146

2147
    def set_error_info(self, pos, used=False):
2148
        self.funcstate.should_declare_error_indicator = True
2149 2150
        if used:
            self.funcstate.uses_error_indicator = True
2151
        if self.code_config.c_line_in_traceback:
2152
            cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
Robert Bradshaw's avatar
Robert Bradshaw committed
2153 2154
        else:
            cinfo = ""
2155

2156
        return "%s = %s[%s]; %s = %s;%s" % (
William Stein's avatar
William Stein committed
2157 2158 2159 2160 2161
            Naming.filename_cname,
            Naming.filetable_cname,
            self.lookup_filename(pos[0]),
            Naming.lineno_cname,
            pos[1],
2162
            cinfo)
2163

2164 2165 2166
    def error_goto(self, pos):
        lbl = self.funcstate.error_label
        self.funcstate.use_label(lbl)
2167 2168 2169
        return "__PYX_ERR(%s, %s, %s)" % (
            self.lookup_filename(pos[0]),
            pos[1],
2170
            lbl)
2171 2172 2173

    def error_goto_if(self, cond, pos):
        return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
2174

Robert Bradshaw's avatar
Robert Bradshaw committed
2175 2176
    def error_goto_if_null(self, cname, pos):
        return self.error_goto_if("!%s" % cname, pos)
2177

Robert Bradshaw's avatar
Robert Bradshaw committed
2178 2179
    def error_goto_if_neg(self, cname, pos):
        return self.error_goto_if("%s < 0" % cname, pos)
2180

Robert Bradshaw's avatar
Robert Bradshaw committed
2181 2182
    def error_goto_if_PyErr(self, pos):
        return self.error_goto_if("PyErr_Occurred()", pos)
2183

William Stein's avatar
William Stein committed
2184
    def lookup_filename(self, filename):
2185
        return self.globalstate.lookup_filename(filename)
William Stein's avatar
William Stein committed
2186

2187
    def put_declare_refcount_context(self):
2188
        self.putln('__Pyx_RefNannyDeclarations')
2189

2190 2191
    def put_setup_refcount_context(self, name, acquire_gil=False):
        if acquire_gil:
2192 2193
            self.globalstate.use_utility_code(
                UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2194
        self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0))
2195

2196
    def put_finish_refcount_context(self):
2197
        self.putln("__Pyx_RefNannyFinishContext();")
2198

2199 2200 2201 2202
    def put_add_traceback(self, qualified_name):
        """
        Build a Python traceback for propagating exceptions.

2203
        qualified_name should be the qualified name of the function.
2204 2205 2206 2207 2208 2209 2210
        """
        format_tuple = (
            qualified_name,
            Naming.clineno_cname,
            Naming.lineno_cname,
            Naming.filename_cname,
        )
2211
        self.funcstate.uses_error_indicator = True
2212 2213
        self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)

2214
    def put_unraisable(self, qualified_name, nogil=False):
2215 2216 2217 2218 2219 2220 2221 2222 2223 2224
        """
        Generate code to print a Python warning for an unraisable exception.

        qualified_name should be the qualified name of the function.
        """
        format_tuple = (
            qualified_name,
            Naming.clineno_cname,
            Naming.lineno_cname,
            Naming.filename_cname,
2225 2226
            self.globalstate.directives['unraisable_tracebacks'],
            nogil,
2227 2228
        )
        self.funcstate.uses_error_indicator = True
2229
        self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %d, %d);' % format_tuple)
2230 2231 2232
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))

2233 2234 2235 2236
    def put_trace_declarations(self):
        self.putln('__Pyx_TraceDeclarations')

    def put_trace_frame_init(self, codeobj=None):
2237 2238
        if codeobj:
            self.putln('__Pyx_TraceFrameInit(%s)' % codeobj)
2239

2240
    def put_trace_call(self, name, pos, nogil=False):
2241 2242
        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d, %s);' % (
            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil, self.error_goto(pos)))
2243

Robert Bradshaw's avatar
Robert Bradshaw committed
2244 2245
    def put_trace_exception(self):
        self.putln("__Pyx_TraceException();")
2246

2247 2248
    def put_trace_return(self, retvalue_cname, nogil=False):
        self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
Robert Bradshaw's avatar
Robert Bradshaw committed
2249

Mark Florisson's avatar
Mark Florisson committed
2250 2251 2252
    def putln_openmp(self, string):
        self.putln("#ifdef _OPENMP")
        self.putln(string)
2253
        self.putln("#endif /* _OPENMP */")
William Stein's avatar
William Stein committed
2254

2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274
    def undef_builtin_expect(self, cond):
        """
        Redefine the macros likely() and unlikely to no-ops, depending on
        condition 'cond'
        """
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   (x)")
        self.putln("    #define unlikely(x) (x)")
        self.putln("#endif")

    def redef_builtin_expect(self, cond):
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   __builtin_expect(!!(x), 1)")
        self.putln("    #define unlikely(x) __builtin_expect(!!(x), 0)")
        self.putln("#endif")

2275
class PyrexCodeWriter(object):
William Stein's avatar
William Stein committed
2276 2277 2278 2279
    # f                file      output file
    # level            int       indentation level

    def __init__(self, outfile_name):
2280
        self.f = Utils.open_new_file(outfile_name)
William Stein's avatar
William Stein committed
2281
        self.level = 0
2282

William Stein's avatar
William Stein committed
2283 2284
    def putln(self, code):
        self.f.write("%s%s\n" % (" " * self.level, code))
2285

William Stein's avatar
William Stein committed
2286 2287
    def indent(self):
        self.level += 1
2288

William Stein's avatar
William Stein committed
2289 2290 2291
    def dedent(self):
        self.level -= 1

2292 2293
class PyxCodeWriter(object):
    """
2294 2295 2296
    Can be used for writing out some Cython code. To use the indenter
    functionality, the Cython.Compiler.Importer module will have to be used
    to load the code to support python 2.4
2297 2298
    """

2299
    def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'):
2300 2301 2302
        self.buffer = buffer or StringIOTree()
        self.level = indent_level
        self.context = context
2303
        self.encoding = encoding
2304 2305 2306

    def indent(self, levels=1):
        self.level += levels
2307
        return True
2308 2309 2310 2311 2312 2313

    def dedent(self, levels=1):
        self.level -= levels

    def indenter(self, line):
        """
2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
        Instead of

            with pyx_code.indenter("for i in range(10):"):
                pyx_code.putln("print i")

        write

            if pyx_code.indenter("for i in range(10);"):
                pyx_code.putln("print i")
                pyx_code.dedent()
2324 2325
        """
        self.putln(line)
2326 2327
        self.indent()
        return True
2328 2329

    def getvalue(self):
2330
        result = self.buffer.getvalue()
2331
        if isinstance(result, bytes):
2332 2333
            result = result.decode(self.encoding)
        return result
2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359

    def putln(self, line, context=None):
        context = context or self.context
        if context:
            line = sub_tempita(line, context)
        self._putln(line)

    def _putln(self, line):
        self.buffer.write("%s%s\n" % (self.level * "    ", line))

    def put_chunk(self, chunk, context=None):
        context = context or self.context
        if context:
            chunk = sub_tempita(chunk, context)

        chunk = textwrap.dedent(chunk)
        for line in chunk.splitlines():
            self._putln(line)

    def insertion_point(self):
        return PyxCodeWriter(self.buffer.insertion_point(), self.level,
                             self.context)

    def named_insertion_point(self, name):
        setattr(self, name, self.insertion_point())

2360 2361

class ClosureTempAllocator(object):
2362
    def __init__(self, klass):
2363 2364 2365 2366 2367 2368 2369 2370 2371 2372
        self.klass = klass
        self.temps_allocated = {}
        self.temps_free = {}
        self.temps_count = 0

    def reset(self):
        for type, cnames in self.temps_allocated.items():
            self.temps_free[type] = list(cnames)

    def allocate_temp(self, type):
2373
        if type not in self.temps_allocated:
2374 2375 2376 2377 2378 2379 2380 2381 2382
            self.temps_allocated[type] = []
            self.temps_free[type] = []
        elif self.temps_free[type]:
            return self.temps_free[type].pop(0)
        cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
        self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
        self.temps_allocated[type].append(cname)
        self.temps_count += 1
        return cname