ModuleNode.py 169 KB
Newer Older
1
#
2
#   Module parse tree node
3 4
#

5 6
from __future__ import absolute_import

7 8
import cython
cython.declare(Naming=object, Options=object, PyrexTypes=object, TypeSlots=object,
9
               error=object, warning=object, py_object_type=object, cy_object_type=object, UtilityCode=object,
10
               EncodedString=object, re=object)
11

12
from collections import defaultdict
13
import json
14
import operator
15
import os
16
import re
da-woods's avatar
da-woods committed
17
import sys
18

19 20 21 22 23 24 25 26 27
from .PyrexTypes import CPtrType
from . import Future
from . import Annotate
from . import Code
from . import Naming
from . import Nodes
from . import Options
from . import TypeSlots
from . import PyrexTypes
28
from . import Pythran
29

30
from .Errors import error, warning
31
from .PyrexTypes import py_object_type, cy_object_type
32
from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version
33
from .Code import UtilityCode, IncludeCode
da-woods's avatar
da-woods committed
34
from .StringEncoding import EncodedString, encoded_string_or_bytes_literal
35
from .Pythran import has_np_pythran
Gary Furnish's avatar
Gary Furnish committed
36

da-woods's avatar
da-woods committed
37 38 39 40 41 42 43 44 45 46 47 48

def replace_suffix_encoded(path, newsuf):
    # calls replace suffix and returns a EncodedString or BytesLiteral with the encoding set
    newpath = replace_suffix(path, newsuf)
    return as_encoded_filename(newpath)

def as_encoded_filename(path):
    # wraps the path with either EncodedString or BytesLiteral (depending on its input type)
    # and sets the encoding to the file system encoding
    return encoded_string_or_bytes_literal(path, sys.getfilesystemencoding())


49 50 51 52
def check_c_declarations_pxd(module_node):
    module_node.scope.check_c_classes_pxd()
    return module_node

53

54
def check_c_declarations(module_node):
55
    module_node.scope.check_c_classes()
56
    module_node.scope.check_c_functions()
57 58
    return module_node

59

60 61 62 63 64 65
def generate_c_code_config(env, options):
    if Options.annotate or options.annotate:
        emit_linenums = False
    else:
        emit_linenums = options.emit_linenums

66 67 68 69
    if hasattr(options, "emit_code_comments"):
        print('Warning: option emit_code_comments is deprecated. '
              'Instead, use compiler directive emit_code_comments.')

70 71
    return Code.CCodeConfig(
        emit_linenums=emit_linenums,
72 73 74
        emit_code_comments=env.directives['emit_code_comments'],
        c_line_in_traceback=options.c_line_in_traceback)

75

76 77 78
class ModuleNode(Nodes.Node, Nodes.BlockNode):
    #  doc       string or None
    #  body      StatListNode
79 80
    #
    #  referenced_modules   [ModuleScope]
81
    #  full_module_name     string
82 83 84
    #
    #  scope                The module scope.
    #  compilation_source   A CompilationSource (see Main)
85
    #  directives           Top-level compiler directives
86

87
    child_attrs = ["body"]
88
    directives = None
89

90 91 92 93 94 95 96 97 98 99 100 101 102 103
    def merge_in(self, tree, scope, merge_scope=False):
        # Merges in the contents of another tree, and possibly scope. With the
        # current implementation below, this must be done right prior
        # to code generation.
        #
        # Note: This way of doing it seems strange -- I believe the
        # right concept is to split ModuleNode into a ModuleNode and a
        # CodeGenerator, and tell that CodeGenerator to generate code
        # from multiple sources.
        assert isinstance(self.body, Nodes.StatListNode)
        if isinstance(tree, Nodes.StatListNode):
            self.body.stats.extend(tree.stats)
        else:
            self.body.stats.append(tree)
104 105 106

        self.scope.utility_code_list.extend(scope.utility_code_list)

107 108 109
        for inc in scope.c_includes.values():
            self.scope.process_include(inc)

110 111 112 113 114 115 116
        def extend_if_not_in(L1, L2):
            for x in L2:
                if x not in L1:
                    L1.append(x)

        extend_if_not_in(self.scope.included_files, scope.included_files)

117
        if merge_scope:
118 119 120
            # Ensure that we don't generate import code for these entries!
            for entry in scope.c_class_entries:
                entry.type.module_name = self.full_module_name
121
                entry.type.scope.directives["internal"] = True
122

123
            self.scope.merge_in(scope)
124

125 126 127 128 129 130 131
    def with_compiler_directives(self):
        # When merging a utility code module into the user code we need to preserve
        # the original compiler directives. This returns the body of the module node,
        # wrapped in its set of directives.
        body = Nodes.CompilerDirectivesNode(self.pos, directives=self.directives, body=self.body)
        return body

132
    def analyse_declarations(self, env):
133 134
        if has_np_pythran(env):
            Pythran.include_pythran_generic(env)
135 136
        if self.directives:
            env.old_style_globals = self.directives['old_style_globals']
137 138 139
        if not Options.docstrings:
            env.doc = self.doc = None
        elif Options.embed_pos_in_docstring:
Robert Bradshaw's avatar
Robert Bradshaw committed
140
            env.doc = EncodedString(u'File: %s (starting at line %s)' % Nodes.relative_position(self.pos))
141
            if self.doc is not None:
142
                env.doc = EncodedString(env.doc + u'\n' + self.doc)
Robert Bradshaw's avatar
Robert Bradshaw committed
143
                env.doc.encoding = self.doc.encoding
144 145
        else:
            env.doc = self.doc
146
        env.directives = self.directives
147
        self.body.analyse_declarations(env)
148

149 150 151 152 153 154
    def prepare_utility_code(self):
        # prepare any utility code that must be created before code generation
        # specifically: CythonUtilityCode
        env = self.scope
        if env.has_import_star:
            self.create_import_star_conversion_utility_code(env)
155 156 157 158
        for name, entry in sorted(env.entries.items()):
            if (entry.create_wrapper and entry.scope is env
                and entry.is_type and entry.type.is_enum):
                    entry.type.create_type_wrapper(env)
159

160 161
    def process_implementation(self, options, result):
        env = self.scope
162
        env.return_type = PyrexTypes.c_void_type
163 164
        self.referenced_modules = []
        self.find_referenced_modules(env, self.referenced_modules, {})
165
        self.sort_cdef_classes(env)
166
        self.generate_c_code(env, options, result)
167
        self.generate_h_code(env, options, result)
168
        self.generate_api_code(env, options, result)
Robert Bradshaw's avatar
Robert Bradshaw committed
169

170 171 172 173 174 175
    def has_imported_c_functions(self):
        for module in self.referenced_modules:
            for entry in module.cfunc_entries:
                if entry.defined_in_pxd:
                    return 1
        return 0
176

177
    def generate_h_code(self, env, options, result):
178
        def h_entries(entries, api=0, pxd=0):
Stefan Behnel's avatar
Stefan Behnel committed
179
            return [entry for entry in entries
180 181 182 183
                    if ((entry.visibility == 'public') or
                        (api and entry.api) or
                        (pxd and entry.defined_in_pxd))]
        h_types = h_entries(env.type_entries, api=1)
Stefan Behnel's avatar
Stefan Behnel committed
184 185 186
        h_vars = h_entries(env.var_entries)
        h_funcs = h_entries(env.cfunc_entries)
        h_extension_types = h_entries(env.c_class_entries)
187
        if h_types or  h_vars or h_funcs or h_extension_types:
da-woods's avatar
da-woods committed
188
            result.h_file = replace_suffix_encoded(result.c_file, ".h")
189
            h_code = Code.CCodeWriter()
190 191
            c_code_config = generate_c_code_config(env, options)
            Code.GlobalState(h_code, self, c_code_config)
192
            if options.generate_pxi:
da-woods's avatar
da-woods committed
193
                result.i_file = replace_suffix_encoded(result.c_file, ".pxi")
194 195 196
                i_code = Code.PyrexCodeWriter(result.i_file)
            else:
                i_code = None
197

198
            h_code.put_generated_by()
da-woods's avatar
da-woods committed
199
            h_guard = self.api_name(Naming.h_guard_prefix, env)
200
            h_code.put_h_guard(h_guard)
201
            h_code.putln("")
202
            h_code.putln('#include "Python.h"')
Stefan Behnel's avatar
Stefan Behnel committed
203
            self.generate_type_header_code(h_types, h_code)
204 205
            if options.capi_reexport_cincludes:
                self.generate_includes(env, [], h_code)
206
            h_code.putln("")
da-woods's avatar
da-woods committed
207
            api_guard = self.api_name(Naming.api_guard_prefix, env)
208
            h_code.putln("#ifndef %s" % api_guard)
209 210
            h_code.putln("")
            self.generate_extern_c_macro_definition(h_code)
211 212
            h_code.putln("")
            self.generate_dl_import_macro(h_code)
Stefan Behnel's avatar
Stefan Behnel committed
213
            if h_extension_types:
214
                h_code.putln("")
Stefan Behnel's avatar
Stefan Behnel committed
215
                for entry in h_extension_types:
216 217 218
                    self.generate_cclass_header_code(entry.type, h_code)
                    if i_code:
                        self.generate_cclass_include_code(entry.type, i_code)
219 220 221 222 223 224 225 226
            if h_funcs:
                h_code.putln("")
                for entry in h_funcs:
                    self.generate_public_declaration(entry, h_code, i_code)
            if h_vars:
                h_code.putln("")
                for entry in h_vars:
                    self.generate_public_declaration(entry, h_code, i_code)
227
            h_code.putln("")
228
            h_code.putln("#endif /* !%s */" % api_guard)
229
            h_code.putln("")
230 231 232
            h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */")
            h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */")
            h_code.putln("")
233
            h_code.putln("#if PY_MAJOR_VERSION < 3")
da-woods's avatar
da-woods committed
234 235 236 237 238 239
            if env.module_name.isascii():
                py2_mod_name = env.module_name
            else:
                py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8")
                h_code.putln('#error "Unicode module names are not supported in Python 2";')
            h_code.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name)
240
            h_code.putln("#else")
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
            py3_mod_func_name = self.mod_init_func_cname('PyInit', env)
            warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % (
                py2_mod_name, py3_mod_func_name, py3_mod_func_name))
            h_code.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.'))
            h_code.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name)
            h_code.putln("")
            h_code.putln("#if PY_VERSION_HEX >= 0x03050000 "
                "&& (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) "
                "|| (defined(__cplusplus) && __cplusplus >= 201402L))")
            h_code.putln("#if defined(__cplusplus) && __cplusplus >= 201402L")
            h_code.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal())
            h_code.putln("#elif defined(__GNUC__) || defined(__clang__)")
            h_code.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % (
                warning_string.as_c_string_literal()))
            h_code.putln("#elif defined(_MSC_VER)")
            h_code.putln('__declspec(deprecated(%s)) __inline' % (
                warning_string.as_c_string_literal()))
            h_code.putln('#endif')
            h_code.putln("static PyObject* __PYX_WARN_IF_INIT_CALLED(PyObject* res) {")
            h_code.putln("return res;")
            h_code.putln("}")
            # Function call is converted to warning macro; uncalled (pointer) is not
            h_code.putln('#define %s() __PYX_WARN_IF_INIT_CALLED(%s())' % (
                py3_mod_func_name, py3_mod_func_name))
            h_code.putln('#endif')
            h_code.putln('#endif')
267 268
            h_code.putln("")
            h_code.putln("#endif /* !%s */" % h_guard)
269

270 271 272 273 274
            f = open_new_file(result.h_file)
            try:
                h_code.copyto(f)
            finally:
                f.close()
275

276 277 278
    def generate_public_declaration(self, entry, h_code, i_code):
        h_code.putln("%s %s;" % (
            Naming.extern_c_macro,
279
            entry.type.declaration_code(entry.cname)))
280
        if i_code:
281 282
            i_code.putln("cdef extern %s" % (
                entry.type.declaration_code(entry.cname, pyrex=1)))
283

da-woods's avatar
da-woods committed
284 285 286
    def api_name(self, prefix, env):
        api_name = self.punycode_module_name(prefix, env.qualified_name)
        return api_name.replace(".", "__")
Robert Bradshaw's avatar
Robert Bradshaw committed
287

288
    def generate_api_code(self, env, options, result):
289
        def api_entries(entries, pxd=0):
290 291 292 293 294 295
            return [entry for entry in entries
                    if entry.api or (pxd and entry.defined_in_pxd)]
        api_vars = api_entries(env.var_entries)
        api_funcs = api_entries(env.cfunc_entries)
        api_extension_types = api_entries(env.c_class_entries)
        if api_vars or api_funcs or api_extension_types:
da-woods's avatar
da-woods committed
296
            result.api_file = replace_suffix_encoded(result.c_file, "_api.h")
297
            h_code = Code.CCodeWriter()
298 299
            c_code_config = generate_c_code_config(env, options)
            Code.GlobalState(h_code, self, c_code_config)
300
            h_code.put_generated_by()
da-woods's avatar
da-woods committed
301
            api_guard = self.api_name(Naming.api_guard_prefix, env)
302
            h_code.put_h_guard(api_guard)
303 304 305 306 307
            # Work around https://bugs.python.org/issue4709
            h_code.putln('#ifdef __MINGW64__')
            h_code.putln('#define MS_WIN64')
            h_code.putln('#endif')

308 309
            h_code.putln('#include "Python.h"')
            if result.h_file:
da-woods's avatar
da-woods committed
310 311 312
                h_filename = os.path.basename(result.h_file)
                h_filename = as_encoded_filename(h_filename)
                h_code.putln('#include %s' % h_filename.as_c_string_literal())
313
            if api_extension_types:
314
                h_code.putln("")
315 316 317 318 319
                for entry in api_extension_types:
                    type = entry.type
                    h_code.putln("static PyTypeObject *%s = 0;" % type.typeptr_cname)
                    h_code.putln("#define %s (*%s)" % (
                        type.typeobj_cname, type.typeptr_cname))
320 321 322 323
            if api_funcs:
                h_code.putln("")
                for entry in api_funcs:
                    type = CPtrType(entry.type)
324
                    cname = env.mangle(Naming.func_prefix_api, entry.name)
325 326 327 328 329 330
                    h_code.putln("static %s = 0;" % type.declaration_code(cname))
                    h_code.putln("#define %s %s" % (entry.name, cname))
            if api_vars:
                h_code.putln("")
                for entry in api_vars:
                    type = CPtrType(entry.type)
331
                    cname = env.mangle(Naming.varptr_prefix_api, entry.name)
332 333
                    h_code.putln("static %s = 0;" %  type.declaration_code(cname))
                    h_code.putln("#define %s (*%s)" % (entry.name, cname))
334
            h_code.put(UtilityCode.load_as_string("PyIdentifierFromString", "ImportExport.c")[0])
335
            if api_vars:
336
                h_code.put(UtilityCode.load_as_string("VoidPtrImport", "ImportExport.c")[1])
337
            if api_funcs:
338
                h_code.put(UtilityCode.load_as_string("FunctionImport", "ImportExport.c")[1])
339
            if api_extension_types:
Robert Bradshaw's avatar
Robert Bradshaw committed
340
                h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[0])
341
                h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[1])
342
            h_code.putln("")
da-woods's avatar
da-woods committed
343
            h_code.putln("static int %s(void) {" % self.api_name("import", env))
344
            h_code.putln("PyObject *module = 0;")
da-woods's avatar
da-woods committed
345
            h_code.putln('module = PyImport_ImportModule(%s);' % env.qualified_name.as_c_string_literal())
346 347
            h_code.putln("if (!module) goto bad;")
            for entry in api_funcs:
348
                cname = env.mangle(Naming.func_prefix_api, entry.name)
349 350
                sig = entry.type.signature_string()
                h_code.putln(
351 352
                    'if (__Pyx_ImportFunction(module, %s, (void (**)(void))&%s, "%s") < 0) goto bad;'
                    % (entry.name.as_c_string_literal(), cname, sig))
353
            for entry in api_vars:
354
                cname = env.mangle(Naming.varptr_prefix_api, entry.name)
355
                sig = entry.type.empty_declaration_code()
356
                h_code.putln(
357 358
                    'if (__Pyx_ImportVoidPtr(module, %s, (void **)&%s, "%s") < 0) goto bad;'
                    % (entry.name.as_c_string_literal(), cname, sig))
359 360
            with ModuleImportGenerator(h_code, imported_modules={env.qualified_name: 'module'}) as import_generator:
                for entry in api_extension_types:
361
                    self.generate_type_import_call(entry.type, h_code, import_generator, error_code="goto bad;")
362
            h_code.putln("Py_DECREF(module); module = 0;")
363 364 365 366 367 368
            h_code.putln("return 0;")
            h_code.putln("bad:")
            h_code.putln("Py_XDECREF(module);")
            h_code.putln("return -1;")
            h_code.putln("}")
            h_code.putln("")
369
            h_code.putln("#endif /* !%s */" % api_guard)
370

371 372 373 374 375
            f = open_new_file(result.api_file)
            try:
                h_code.copyto(f)
            finally:
                f.close()
376

377
    def generate_cclass_header_code(self, type, h_code):
378
        h_code.putln("%s %s %s;" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
379
            Naming.extern_c_macro,
380
            PyrexTypes.public_decl("PyTypeObject", "DL_IMPORT"),
381
            type.typeobj_cname))
382

383 384 385 386 387 388 389
    def generate_cclass_include_code(self, type, i_code):
        i_code.putln("cdef extern class %s.%s:" % (
            type.module_name, type.name))
        i_code.indent()
        var_entries = type.scope.var_entries
        if var_entries:
            for entry in var_entries:
390 391
                i_code.putln("cdef %s" % (
                    entry.type.declaration_code(entry.cname, pyrex=1)))
392 393 394
        else:
            i_code.putln("pass")
        i_code.dedent()
395

396
    def generate_c_code(self, env, options, result):
397
        modules = self.referenced_modules
398

399
        if Options.annotate or options.annotate:
400 401
            show_entire_c_code = Options.annotate == "fullc" or options.annotate == "fullc"
            rootwriter = Annotate.AnnotationCCodeWriter(show_entire_c_code=show_entire_c_code)
402
        else:
403 404
            rootwriter = Code.CCodeWriter()

405 406
        c_code_config = generate_c_code_config(env, options)

407 408 409 410 411
        globalstate = Code.GlobalState(
            rootwriter, self,
            code_config=c_code_config,
            common_utility_include_dir=options.common_utility_include_dir,
        )
412
        globalstate.initialize_main_c_code()
413
        h_code = globalstate['h_code']
414

415
        self.generate_module_preamble(env, options, modules, result.embedded_metadata, h_code)
416

417 418
        globalstate.module_pos = self.pos
        globalstate.directives = self.directives
419

420
        globalstate.use_utility_code(refnanny_utility_code)
421

422
        code = globalstate['before_global_var']
da-woods's avatar
da-woods committed
423 424 425
        code.putln('#define __Pyx_MODULE_NAME %s' %
                   self.full_module_name.as_c_string_literal())
        module_is_main = self.is_main_module_flag_cname()
426 427
        code.putln("extern int %s;" % module_is_main)
        code.putln("int %s = 0;" % module_is_main)
428
        code.putln("")
da-woods's avatar
da-woods committed
429
        code.putln("/* Implementation of %s */" % env.qualified_name.as_c_string_literal())
430

Jeroen Demeyer's avatar
Jeroen Demeyer committed
431 432 433
        code = globalstate['late_includes']
        self.generate_includes(env, modules, code, early=False)

434
        code = globalstate['module_code']
435

436
        self.generate_cached_builtins_decls(env, code)
437

438
        # generate normal variable and function definitions
439
        self.generate_lambda_definitions(env, code)
440
        self.generate_variable_definitions(env, code)
441
        self.body.generate_function_definitions(env, code)
442

Robert Bradshaw's avatar
Robert Bradshaw committed
443
        code.mark_pos(None)
444 445
        self.generate_typeobj_definitions(env, code)
        self.generate_method_table(env, code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
446 447
        if env.has_import_star:
            self.generate_import_star(env, code)
448

449 450 451
        # initialise the macro to reduce the code size of one-time functionality
        code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip())

452 453 454 455 456
        self.generate_module_state_start(env, globalstate['module_state'])
        self.generate_module_state_defines(env, globalstate['module_state_defines'])
        self.generate_module_state_clear(env, globalstate['module_state_clear'])
        self.generate_module_state_traverse(env, globalstate['module_state_traverse'])

457 458 459
        # init_globals is inserted before this
        self.generate_module_init_func(modules[:-1], env, globalstate['init_module'])
        self.generate_module_cleanup_func(env, globalstate['cleanup_module'])
460
        if Options.embed:
461 462
            self.generate_main_method(env, globalstate['main_method'])
        self.generate_filename_table(globalstate['filename_table'])
463

464
        self.generate_declarations_for_modules(env, modules, globalstate)
465
        h_code.write('\n')
Gary Furnish's avatar
Gary Furnish committed
466

467
        for utilcode in env.utility_code_list[:]:
468
            globalstate.use_utility_code(utilcode)
469
        globalstate.finalize_main_c_code()
470

471 472
        self.generate_module_state_end(env, modules, globalstate)

473
        f = open_new_file(result.c_file)
Stefan Behnel's avatar
Stefan Behnel committed
474 475 476 477
        try:
            rootwriter.copyto(f)
        finally:
            f.close()
478
        result.c_file_generated = 1
Stefan Behnel's avatar
Stefan Behnel committed
479 480
        if options.gdb_debug:
            self._serialize_lineno_map(env, rootwriter)
481
        if Options.annotate or options.annotate:
482
            self._generate_annotations(rootwriter, result, options)
483

484
    def _generate_annotations(self, rootwriter, result, options):
485
        self.annotate(rootwriter)
486 487 488 489 490 491 492 493 494 495 496 497 498 499

        coverage_xml_filename = Options.annotate_coverage_xml or options.annotate_coverage_xml
        if coverage_xml_filename and os.path.exists(coverage_xml_filename):
            try:
                import xml.etree.cElementTree as ET
            except ImportError:
                import xml.etree.ElementTree as ET
            coverage_xml = ET.parse(coverage_xml_filename).getroot()
            for el in coverage_xml.getiterator():
                el.tail = None  # save some memory
        else:
            coverage_xml = None

        rootwriter.save_annotation(result.main_source_file, result.c_file, coverage_xml=coverage_xml)
500 501

        # if we included files, additionally generate one annotation file for each
502 503 504
        if not self.scope.included_files:
            return

505
        search_include_file = self.scope.context.search_include_directories
506
        target_dir = os.path.abspath(os.path.dirname(result.c_file))
507
        for included_file in self.scope.included_files:
508 509 510 511 512
            target_file = os.path.abspath(os.path.join(target_dir, included_file))
            target_file_dir = os.path.dirname(target_file)
            if not target_file_dir.startswith(target_dir):
                # any other directories may not be writable => avoid trying
                continue
513 514 515 516 517 518
            source_file = search_include_file(included_file, "", self.pos, include=True)
            if not source_file:
                continue
            if target_file_dir != target_dir and not os.path.exists(target_file_dir):
                try:
                    os.makedirs(target_file_dir)
519
                except OSError as e:
520 521 522
                    import errno
                    if e.errno != errno.EEXIST:
                        raise
523
            rootwriter.save_annotation(source_file, target_file, coverage_xml=coverage_xml)
524

Mark Florisson's avatar
Mark Florisson committed
525
    def _serialize_lineno_map(self, env, ccodewriter):
526
        tb = env.context.gdb_debug_outputwriter
Mark Florisson's avatar
Mark Florisson committed
527
        markers = ccodewriter.buffer.allmarkers()
528

529
        d = defaultdict(list)
530
        for c_lineno, cython_lineno in enumerate(markers):
Mark Florisson's avatar
Mark Florisson committed
531
            if cython_lineno > 0:
532
                d[cython_lineno].append(c_lineno + 1)
533

Mark Florisson's avatar
Mark Florisson committed
534
        tb.start('LineNumberMapping')
535
        for cython_lineno, c_linenos in sorted(d.items()):
536 537 538 539 540
            tb.add_entry(
                'LineNumber',
                c_linenos=' '.join(map(str, c_linenos)),
                cython_lineno=str(cython_lineno),
            )
Mark Florisson's avatar
Mark Florisson committed
541 542
        tb.end('LineNumberMapping')
        tb.serialize()
543

544 545 546 547 548 549
    def find_referenced_modules(self, env, module_list, modules_seen):
        if env not in modules_seen:
            modules_seen[env] = 1
            for imported_module in env.cimported_modules:
                self.find_referenced_modules(imported_module, module_list, modules_seen)
            module_list.append(env)
Gary Furnish's avatar
Gary Furnish committed
550

551
    def sort_types_by_inheritance(self, type_dict, type_order, getkey):
552 553 554
        # copy the types into a list moving each parent type before
        # its first child
        type_list = []
555 556
        for i, key in enumerate(type_order):
            new_entry = type_dict[key]
557 558

            # collect all base classes to check for children
559
            hierarchy = set()
560
            base = new_entry
561 562 563 564 565 566 567
            while base:
                base_type = base.type.base_type
                if not base_type:
                    break
                base_key = getkey(base_type)
                hierarchy.add(base_key)
                base = type_dict.get(base_key)
568
            new_entry.base_keys = hierarchy
569

570
            # find the first (sub-)subclass and insert before that
571 572
            for j in range(i):
                entry = type_list[j]
573
                if key in entry.base_keys:
574 575 576 577 578 579 580
                    type_list.insert(j, new_entry)
                    break
            else:
                type_list.append(new_entry)
        return type_list

    def sort_type_hierarchy(self, module_list, env):
581 582 583
        # poor developer's OrderedDict
        vtab_dict, vtab_dict_order = {}, []
        vtabslot_dict, vtabslot_dict_order = {}, []
Stefan Behnel's avatar
Stefan Behnel committed
584

585 586 587 588 589 590 591 592 593
        def vtab_key_func(entry_type):
            return entry_type.vtabstruct_cname

        def vtabslot_key_func(entry_type):
            if entry_type.is_cyp_wrapper:
                # cyp_wrappers all have the same objstruct_cname
                return entry_type.wrapped_cname
            return entry_type.objstruct_cname

Gary Furnish's avatar
Gary Furnish committed
594 595
        for module in module_list:
            for entry in module.c_class_entries:
596
                if entry.used and not entry.in_cinclude:
Gary Furnish's avatar
Gary Furnish committed
597
                    type = entry.type
598
                    key = vtab_key_func(type)
599 600
                    if not key:
                        continue
Stefan Behnel's avatar
Stefan Behnel committed
601 602 603
                    if key in vtab_dict:
                        # FIXME: this should *never* happen, but apparently it does
                        # for Cython generated utility code
604
                        from .UtilityCode import NonManglingModuleScope
Stefan Behnel's avatar
Stefan Behnel committed
605 606 607 608 609
                        assert isinstance(entry.scope, NonManglingModuleScope), str(entry.scope)
                        assert isinstance(vtab_dict[key].scope, NonManglingModuleScope), str(vtab_dict[key].scope)
                    else:
                        vtab_dict[key] = entry
                        vtab_dict_order.append(key)
610 611
            all_defined_here = module is env
            for entry in module.type_entries:
612
                if entry.used and (all_defined_here or entry.defined_in_pxd):
Gary Furnish's avatar
Gary Furnish committed
613
                    type = entry.type
614 615
                    if type.is_extension_type and not entry.in_cinclude:
                        type = entry.type
616
                        key = vtabslot_key_func(type)
617 618 619
                        assert key not in vtabslot_dict, key
                        vtabslot_dict[key] = entry
                        vtabslot_dict_order.append(key)
620

621
        vtab_list = self.sort_types_by_inheritance(
622
            vtab_dict, vtab_dict_order, vtab_key_func)
623

624
        vtabslot_list = self.sort_types_by_inheritance(
625
            vtabslot_dict, vtabslot_dict_order, vtabslot_key_func)
626

627
        return (vtab_list, vtabslot_list)
628

629
    def sort_cdef_classes(self, env):
630 631 632 633 634 635
        def key_func(entry_type):
            if entry_type.is_cyp_wrapper:
                # cyp_wrappers all have the same objstruct_cname
                return entry_type.wrapped_cname
            return entry_type.objstruct_cname

636
        entry_dict, entry_order = {}, []
637
        for entry in env.c_class_entries:
Stefan Behnel's avatar
Stefan Behnel committed
638
            key = key_func(entry.type)
639
            assert key not in entry_dict, key
Stefan Behnel's avatar
Stefan Behnel committed
640
            entry_dict[key] = entry
641
            entry_order.append(key)
642
        env.c_class_entries[:] = self.sort_types_by_inheritance(
643
            entry_dict, entry_order, key_func)
644

645
    def generate_type_definitions(self, env, modules, vtab_list, vtabslot_list, code, globalstate):
646 647 648
        # TODO: Why are these separated out?
        for entry in vtabslot_list:
            self.generate_objstruct_predeclaration(entry.type, code)
649
        vtabslot_entries = set(vtabslot_list)
650
        ctuple_names = set()
Gary Furnish's avatar
Gary Furnish committed
651 652
        for module in modules:
            definition = module is env
653 654
            type_entries = []
            for entry in module.type_entries:
655
                if entry.type.is_ctuple and entry.used:
656 657
                    if entry.name not in ctuple_names:
                        ctuple_names.add(entry.name)
Gary Furnish's avatar
Gary Furnish committed
658
                        type_entries.append(entry)
659 660
                elif definition or entry.defined_in_pxd:
                    type_entries.append(entry)
661
            type_entries = [t for t in type_entries if t not in vtabslot_entries]
662
            self.generate_type_header_code(type_entries, code)
663
            code.putln("")
664 665 666
            code.putln("/* PyTypeObject pointer declarations for all c classes */")
            self.generate_c_class_declarations(module, code, definition, globalstate)

Gary Furnish's avatar
Gary Furnish committed
667
        for entry in vtabslot_list:
668
            self.generate_objstruct_definition(entry.type, code)
669
            self.generate_typeobj_predeclaration(entry, code)
Gary Furnish's avatar
Gary Furnish committed
670
        for entry in vtab_list:
671
            self.generate_typeobj_predeclaration(entry, code)
Gary Furnish's avatar
Gary Furnish committed
672 673
            self.generate_exttype_vtable_struct(entry, code)
            self.generate_exttype_vtabptr_declaration(entry, code)
674
            self.generate_exttype_final_methods_declaration(entry, code)
675 676

        from .CypclassWrapper import generate_cyp_class_deferred_definitions
677 678 679 680
        for module in modules:
            definition = module is env
            code.putln("")
            code.putln("/* Deferred definitions for cypclasses */")
681
            generate_cyp_class_deferred_definitions(env, code, definition)
Gary Furnish's avatar
Gary Furnish committed
682

683 684 685
    def generate_declarations_for_modules(self, env, modules, globalstate):
        typecode = globalstate['type_declarations']
        typecode.putln("")
686
        typecode.putln("/*--- Type declarations ---*/")
687 688 689 690 691 692 693
        # This is to work around the fact that array.h isn't part of the C-API,
        # but we need to declare it earlier than utility code.
        if 'cpython.array' in [m.qualified_name for m in modules]:
            typecode.putln('#ifndef _ARRAYARRAY_H')
            typecode.putln('struct arrayobject;')
            typecode.putln('typedef struct arrayobject arrayobject;')
            typecode.putln('#endif')
694 695
        vtab_list, vtabslot_list = self.sort_type_hierarchy(modules, env)
        self.generate_type_definitions(
696
            env, modules, vtab_list, vtabslot_list, typecode, globalstate)
697
        modulecode = globalstate['module_declarations']
Gary Furnish's avatar
Gary Furnish committed
698
        for module in modules:
699
            defined_here = module is env
700
            modulecode.putln("")
da-woods's avatar
da-woods committed
701
            modulecode.putln("/* Module declarations from %s */" % module.qualified_name.as_c_string_literal())
702 703
            self.generate_cvariable_declarations(module, modulecode, defined_here)
            self.generate_cfunction_declarations(module, modulecode, defined_here)
Gary Furnish's avatar
Gary Furnish committed
704

705 706 707
    def _put_setup_code(self, code, name):
        code.put(UtilityCode.load_as_string(name, "ModuleSetupCode.c")[1])

708
    def generate_module_preamble(self, env, options, cimported_modules, metadata, code):
709
        code.put_generated_by()
710
        if metadata:
711
            code.putln("/* BEGIN: Cython Metadata")
712
            code.putln(json.dumps(metadata, indent=4, sort_keys=True))
713
            code.putln("END: Cython Metadata */")
714
            code.putln("")
715

716
        code.putln("#define PY_SSIZE_T_CLEAN")
717
        self._put_setup_code(code, "InitLimitedAPI")
718

719 720 721
        for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey):
            if inc.location == inc.INITIAL:
                inc.write(code)
722
        code.putln("#ifndef Py_PYTHON_H")
723 724
        code.putln("    #error Python headers needed to compile C extensions, "
                   "please install development version of Python.")
725
        code.putln("#elif PY_VERSION_HEX < 0x02070000 || "
726
                   "(0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000)")
727
        code.putln("    #error Cython requires Python 2.7+ or Python 3.3+.")
728 729
        code.putln("#else")
        code.globalstate["end"].putln("#endif /* Py_PYTHON_H */")
Robert Bradshaw's avatar
Robert Bradshaw committed
730

731
        from .. import __version__
Robert Bradshaw's avatar
Robert Bradshaw committed
732
        code.putln('#define CYTHON_ABI "%s"' % __version__.replace('.', '_'))
733
        code.putln('#define CYTHON_HEX_VERSION %s' % build_hex_version(__version__))
734 735
        code.putln("#define CYTHON_FUTURE_DIVISION %d" % (
            Future.division in env.context.future_directives))
736

737 738 739 740 741
        self._put_setup_code(code, "CModulePreamble")
        if env.context.options.cplus:
            self._put_setup_code(code, "CppInitCode")
        else:
            self._put_setup_code(code, "CInitCode")
742
        self._put_setup_code(code, "PythonCompatibility")
743
        self._put_setup_code(code, "MathInitCode")
744

745
        # Using "(void)cname" to prevent "unused" warnings.
746
        if options.c_line_in_traceback:
747
            cinfo = "%s = %s; (void)%s; " % (Naming.clineno_cname, Naming.line_c_macro, Naming.clineno_cname)
748 749
        else:
            cinfo = ""
750 751 752 753 754 755 756 757
        code.putln("#define __PYX_MARK_ERR_POS(f_index, lineno) \\")
        code.putln("    { %s = %s[f_index]; (void)%s; %s = lineno; (void)%s; %s}" % (
            Naming.filename_cname, Naming.filetable_cname, Naming.filename_cname,
            Naming.lineno_cname, Naming.lineno_cname,
            cinfo
        ))
        code.putln("#define __PYX_ERR(f_index, lineno, Ln_error) \\")
        code.putln("    { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; }")
758

759
        code.putln("")
760
        self.generate_extern_c_macro_definition(code)
761
        code.putln("")
762

da-woods's avatar
da-woods committed
763 764
        code.putln("#define %s" % self.api_name(Naming.h_guard_prefix, env))
        code.putln("#define %s" % self.api_name(Naming.api_guard_prefix, env))
Jeroen Demeyer's avatar
Jeroen Demeyer committed
765 766
        code.putln("/* Early includes */")
        self.generate_includes(env, cimported_modules, code, late=False)
767
        code.putln("")
768
        code.putln("#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS)")
769 770 771
        code.putln("#define CYTHON_WITHOUT_ASSERTIONS")
        code.putln("#endif")
        code.putln("")
772

773 774 775 776
        if env.directives['ccomplex']:
            code.putln("")
            code.putln("#if !defined(CYTHON_CCOMPLEX)")
            code.putln("#define CYTHON_CCOMPLEX 1")
777
            code.putln("#endif")
778
            code.putln("")
779
        code.put(UtilityCode.load_as_string("UtilityFunctionPredeclarations", "ModuleSetupCode.c")[0])
780 781 782

        c_string_type = env.directives['c_string_type']
        c_string_encoding = env.directives['c_string_encoding']
783 784
        if c_string_type not in ('bytes', 'bytearray') and not c_string_encoding:
            error(self.pos, "a default encoding must be provided if c_string_type is not a byte type")
785
        code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII %s' % int(c_string_encoding == 'ascii'))
786 787
        code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 %s' %
                int(c_string_encoding.replace('-', '').lower() == 'utf8'))
788 789 790
        if c_string_encoding == 'default':
            code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 1')
        else:
791 792
            code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT '
                    '(PY_MAJOR_VERSION >= 3 && __PYX_DEFAULT_STRING_ENCODING_IS_UTF8)')
793
            code.putln('#define __PYX_DEFAULT_STRING_ENCODING "%s"' % c_string_encoding)
794 795 796 797 798 799
        if c_string_type == 'bytearray':
            c_string_func_name = 'ByteArray'
        else:
            c_string_func_name = c_string_type.title()
        code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_func_name)
        code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_func_name)
800
        code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0])
Robert Bradshaw's avatar
Robert Bradshaw committed
801

802 803 804
        # These utility functions are assumed to exist and used elsewhere.
        PyrexTypes.c_long_type.create_to_py_utility_code(env)
        PyrexTypes.c_long_type.create_from_py_utility_code(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
805
        PyrexTypes.c_int_type.create_from_py_utility_code(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
806

Robert Bradshaw's avatar
Robert Bradshaw committed
807
        code.put(Nodes.branch_prediction_macros)
808
        code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }')
809
        code.putln('')
810
        code.putln('#if !CYTHON_COMPILING_IN_LIMITED_API')
811
        code.putln('static PyObject *%s = NULL;' % env.module_cname)
812
        code.putln('static PyObject *%s;' % env.module_dict_cname)
813
        code.putln('static PyObject *%s;' % Naming.builtins_cname)
814
        code.putln('static PyObject *%s = NULL;' % Naming.cython_runtime_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
815
        code.putln('static PyObject *%s;' % Naming.empty_tuple)
Robert Bradshaw's avatar
Robert Bradshaw committed
816
        code.putln('static PyObject *%s;' % Naming.empty_bytes)
817
        code.putln('static PyObject *%s;' % Naming.empty_unicode)
818 819
        if Options.pre_import is not None:
            code.putln('static PyObject *%s;' % Naming.preimport_cname)
820 821
        code.putln('#endif')

822
        code.putln('static int %s;' % Naming.lineno_cname)
823
        code.putln('static int %s = 0;' % Naming.clineno_cname)
Stefan Behnel's avatar
Stefan Behnel committed
824
        code.putln('static const char * %s = %s;' % (Naming.cfilenm_cname, Naming.file_c_macro))
825
        code.putln('static const char *%s;' % Naming.filename_cname)
826

827
        env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c"))
828 829 830
        if has_np_pythran(env):
            env.use_utility_code(UtilityCode.load_cached("PythranConversion", "CppSupport.cpp"))

831 832
    def generate_extern_c_macro_definition(self, code):
        name = Naming.extern_c_macro
833 834 835 836 837 838
        code.putln("#ifndef %s" % name)
        code.putln("  #ifdef __cplusplus")
        code.putln('    #define %s extern "C"' % name)
        code.putln("  #else")
        code.putln("    #define %s extern" % name)
        code.putln("  #endif")
839 840
        code.putln("#endif")

841 842 843 844 845
    def generate_dl_import_macro(self, code):
        code.putln("#ifndef DL_IMPORT")
        code.putln("  #define DL_IMPORT(_T) _T")
        code.putln("#endif")

Jeroen Demeyer's avatar
Jeroen Demeyer committed
846
    def generate_includes(self, env, cimported_modules, code, early=True, late=True):
847 848 849 850 851 852 853
        for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey):
            if inc.location == inc.EARLY:
                if early:
                    inc.write(code)
            elif inc.location == inc.LATE:
                if late:
                    inc.write(code)
Jeroen Demeyer's avatar
Jeroen Demeyer committed
854 855
        if early:
            code.putln_openmp("#include <omp.h>")
856

857
    def generate_filename_table(self, code):
858
        from os.path import isabs, basename
859
        code.putln("")
Robert Bradshaw's avatar
Robert Bradshaw committed
860
        code.putln("static const char *%s[] = {" % Naming.filetable_cname)
861 862
        if code.globalstate.filename_list:
            for source_desc in code.globalstate.filename_list:
863 864 865 866
                file_path = source_desc.get_filenametable_entry()
                if isabs(file_path):
                    file_path = basename(file_path)  # never include absolute paths
                escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"')
da-woods's avatar
da-woods committed
867 868
                escaped_filename = as_encoded_filename(escaped_filename)
                code.putln('%s,' % escaped_filename.as_c_string_literal())
869 870 871 872
        else:
            # Some C compilers don't like an empty array
            code.putln("0")
        code.putln("};")
873 874 875 876

    def generate_type_predeclarations(self, env, code):
        pass

877
    def generate_type_header_code(self, type_entries, code):
878 879
        # Generate definitions of structs/unions/enums/typedefs/objstructs.
        #self.generate_gcc33_hack(env, code) # Is this still needed?
880
        # Forward declarations
881
        for entry in type_entries:
882
            if not entry.in_cinclude:
883
                #print "generate_type_header_code:", entry.name, repr(entry.type) ###
884
                type = entry.type
885
                if type.is_typedef: # Must test this first!
886
                    pass
887
                elif type.is_struct_or_union or type.is_cpp_class:
888
                    self.generate_struct_union_predeclaration(entry, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
889 890
                elif type.is_ctuple and entry.used:
                    self.generate_struct_union_predeclaration(entry.type.struct_entry, code)
891 892 893 894 895 896 897 898 899
                elif type.is_extension_type:
                    self.generate_objstruct_predeclaration(type, code)
        # Actual declarations
        for entry in type_entries:
            if not entry.in_cinclude:
                #print "generate_type_header_code:", entry.name, repr(entry.type) ###
                type = entry.type
                if type.is_typedef: # Must test this first!
                    self.generate_typedef(entry, code)
900
                elif type.is_enum:
901
                    self.generate_enum_definition(entry, code)
902 903
                elif type.is_struct_or_union:
                    self.generate_struct_union_definition(entry, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
904 905
                elif type.is_ctuple and entry.used:
                    self.generate_struct_union_definition(entry.type.struct_entry, code)
906 907
                elif type.is_cpp_class:
                    self.generate_cpp_class_definition(entry, code)
908
                elif type.is_extension_type:
909
                    self.generate_objstruct_definition(type, code)
910

911 912 913 914 915 916 917 918 919 920 921 922 923
    def generate_gcc33_hack(self, env, code):
        # Workaround for spurious warning generation in gcc 3.3
        code.putln("")
        for entry in env.c_class_entries:
            type = entry.type
            if not type.typedef_flag:
                name = type.objstruct_cname
                if name.startswith("__pyx_"):
                    tail = name[6:]
                else:
                    tail = name
                code.putln("typedef struct %s __pyx_gcc33_%s;" % (
                    name, tail))
924

925 926
    def generate_typedef(self, entry, code):
        base_type = entry.type.typedef_base_type
927 928
        enclosing_scope = entry.scope
        if base_type.is_numeric and not enclosing_scope.is_cpp_class_scope:
929 930 931 932
            try:
                writer = code.globalstate['numeric_typedefs']
            except KeyError:
                writer = code
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
933 934
        else:
            writer = code
935
        writer.mark_pos(entry.pos)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
936
        writer.putln("typedef %s;" % base_type.declaration_code(entry.cname))
937

938
    def sue_predeclaration(self, type, kind, name):
939
        if type.typedef_flag:
940 941 942
            return "%s %s;\ntypedef %s %s %s;" % (
                kind, name,
                kind, name, name)
943
        else:
944 945 946 947
            return "%s %s;" % (kind, name)

    def generate_struct_union_predeclaration(self, entry, code):
        type = entry.type
948
        if type.is_cpp_class and type.templates:
949 950
            code.putln("template <typename %s>" % ", typename ".join(
                [T.empty_declaration_code() for T in type.templates]))
951 952 953 954 955
        code.putln(self.sue_predeclaration(type, type.kind, type.cname))

    def sue_header_footer(self, type, kind, name):
        header = "%s %s {" % (kind, name)
        footer = "};"
956
        return header, footer
957

958
    def generate_struct_union_definition(self, entry, code):
959
        code.mark_pos(entry.pos)
960 961 962
        type = entry.type
        scope = type.scope
        if scope:
963 964 965 966 967
            kind = type.kind
            packed = type.is_struct and type.packed
            if packed:
                kind = "%s %s" % (type.kind, "__Pyx_PACKED")
                code.globalstate.use_utility_code(packed_struct_utility_code)
968
            header, footer = \
969 970
                self.sue_header_footer(type, kind, type.cname)
            if packed:
971 972 973 974
                code.putln("#if defined(__SUNPRO_C)")
                code.putln("  #pragma pack(1)")
                code.putln("#elif !defined(__GNUC__)")
                code.putln("  #pragma pack(push, 1)")
975
                code.putln("#endif")
976 977 978
            code.putln(header)
            var_entries = scope.var_entries
            if not var_entries:
979
                error(entry.pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block")
980 981
            for attr in var_entries:
                code.putln(
982
                    "%s;" % attr.type.declaration_code(attr.cname))
983
            code.putln(footer)
984
            if packed:
985 986 987 988
                code.putln("#if defined(__SUNPRO_C)")
                code.putln("  #pragma pack()")
                code.putln("#elif !defined(__GNUC__)")
                code.putln("  #pragma pack(pop)")
989
                code.putln("#endif")
990

991 992 993 994
    def generate_cpp_class_definition(self, entry, code):
        code.mark_pos(entry.pos)
        type = entry.type
        scope = type.scope
gsamain's avatar
gsamain committed
995
        default_constructor = False
996 997
        if scope:
            if type.templates:
998 999
                code.putln("template <class %s>" % ", class ".join(
                    [T.empty_declaration_code() for T in type.templates]))
1000 1001 1002
            # Just let everything be public.
            code.put("struct %s" % type.cname)
            if type.base_classes:
1003
                base_class_list = [base_class.empty_declaration_code() for base_class in type.base_classes]
1004 1005 1006 1007 1008 1009 1010 1011
                # if type.is_cyp_class and (type.base_classes[-1] is cy_object_type or type.base_classes[-1].name == "ActhonActivableClass"):
                #     base_class_list[-1] = "virtual " + base_class_list[-1]
                if type.is_cyp_class:
                    base_class_decl = ",virtual public ".join(base_class_list)
                    code.put(" : virtual public %s" % base_class_decl)
                else:
                    base_class_decl = ", public ".join(base_class_list)
                    code.put(" : public %s" % base_class_decl)
1012
            code.putln(" {")
1013
            self.generate_type_header_code(scope.type_entries, code)
1014 1015
            py_attrs = [e for e in scope.entries.values()
                        if e.type.is_pyobject and not e.is_inherited]
1016
            cypclass_attrs = [e for e in scope.var_entries
1017 1018
                        if e.type.is_cyp_class and not e.name == "this"
                        and not e.is_type]
1019
            has_virtual_methods = False
1020 1021
            constructor = None
            destructor = None
1022 1023
            if entry.type.is_cyp_class and entry.type.activable:
                activated_class_entry = scope.lookup_here("Activated")
1024
                code.putln("struct %s;" % activated_class_entry.cname)
1025 1026
                dunder_activate_entry = scope.lookup_here("__activate__")
                code.putln("%s;" % dunder_activate_entry.type.declaration_code(dunder_activate_entry.cname))
1027
            for attr in scope.var_entries:
1028
                cname = attr.cname
1029 1030
                if attr.type.is_cfunction and attr.type.is_static_method:
                    code.put("static ")
1031
                elif attr.name == "<init>":
1032 1033
                    #constructor = attr
                    constructor = scope.lookup_here("<init>")
1034 1035 1036
                elif attr.name == "<del>":
                    destructor = attr
                elif attr.type.is_cfunction:
1037
                    code.put("virtual ")
1038
                    has_virtual_methods = True
gsamain's avatar
gsamain committed
1039 1040 1041
                    if 'operator ' in attr.name:
                        code.putln("%s();" % attr.cname)
                        continue
1042 1043
                elif attr.type.is_cyp_class:
                    cname = "%s = NULL" % cname
1044
                code.putln("%s;" % attr.type.declaration_code(cname))
1045 1046 1047
                if type.is_cyp_class and attr.type.is_cfunction and attr.type.is_static_method and attr.static_cname is not None:
                    self.generate_cyp_class_static_method_resolution(attr, code)

1048
            is_implementing = 'init_module' in code.globalstate.parts
1049

1050 1051 1052
            for reified in scope.reifying_entries:
                code.putln("struct %s;" % reified.cname)

1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
            def generate_cpp_constructor_code(arg_decls, arg_names, is_implementing, py_attrs, constructor):
                if is_implementing:
                    code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls)))
                    if py_attrs:
                        code.put_ensure_gil()
                        for attr in py_attrs:
                            code.put_init_var_to_py_none(attr, nanny=False);
                    if constructor:
                        code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names)))
                    if py_attrs:
                        code.put_release_ensured_gil()
                    code.putln("}")
                else:
                    code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls)))

1068 1069 1070 1071
            if type.is_cyp_class:
                constructor = scope.lookup_here("<constructor>")
                for constructor_alternative in constructor.all_alternatives():
                    code.putln("static %s;" % constructor_alternative.type.declaration_code(constructor_alternative.cname))
1072
                self.generate_cyp_class_mro_method_resolution(scope, code)
1073
            elif constructor or py_attrs:
1074
                if constructor:
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
                    for constructor_alternative in constructor.all_alternatives():
                        arg_decls = []
                        arg_names = []
                        for arg in constructor_alternative.type.original_args[
                                :len(constructor_alternative.type.args)-constructor_alternative.type.optional_arg_count]:
                            arg_decls.append(arg.declaration_code())
                            arg_names.append(arg.cname)
                        if constructor_alternative.type.optional_arg_count:
                            arg_decls.append(constructor_alternative.type.op_arg_struct.declaration_code(Naming.optional_args_cname))
                            arg_names.append(Naming.optional_args_cname)
                        if not arg_decls:
                            default_constructor = True
                            arg_decls = ["void"]
                        generate_cpp_constructor_code(arg_decls, arg_names, is_implementing, py_attrs, constructor_alternative)
1089 1090 1091
                else:
                    arg_decls = ["void"]
                    arg_names = []
1092 1093
                    generate_cpp_constructor_code(arg_decls, arg_names, is_implementing, py_attrs, constructor)

1094 1095 1096 1097 1098 1099 1100 1101
            if type.is_cyp_class and cypclass_attrs:
                # Declaring a small destruction handler which will always try to Cy_XDECREF
                # every cypclass attribute. This handler is defined after all class definition.
                # We cannot define it inplace, because we won't be able to decref forward declare attributes
                # (as they're not defined, they're not considered CyObject subclasses, so Cy_DECREF will be lost)
                cypclass_attrs_destructor_name = "%s__cypclass_attrs_destructor__%s" % (Naming.func_prefix, type.name)
                code.putln("void " + cypclass_attrs_destructor_name + "();")

gsamain's avatar
gsamain committed
1102 1103
            if type.is_cyp_class or destructor or py_attrs or has_virtual_methods:
                if has_virtual_methods or type.is_cyp_class:
1104
                    code.put("virtual ")
1105 1106
                if is_implementing:
                  code.putln("~%s() {" % type.cname)
1107 1108 1109
                  if cypclass_attrs:
                      cypclass_attrs_destructor_name = "%s__cypclass_attrs_destructor__%s" % (Naming.func_prefix, entry.name)
                      code.putln(cypclass_attrs_destructor_name + "();")
1110 1111 1112 1113 1114 1115
                  if py_attrs:
                      code.put_ensure_gil()
                  if destructor:
                      code.putln("%s();" % destructor.cname)
                  if py_attrs:
                      for attr in py_attrs:
1116
                          code.put_var_xdecref(attr, nanny=False)
1117 1118 1119 1120
                      code.put_release_ensured_gil()
                  code.putln("}")
                else:
                  code.putln("~%s();" % type.cname)
1121 1122
            if py_attrs:
                # Also need copy constructor and assignment operators.
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
                if is_implementing:
                  code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname))
                  code.put_ensure_gil()
                  for attr in scope.var_entries:
                      if not attr.type.is_cfunction:
                          code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname))
                          code.put_var_incref(attr, nanny=False)
                  code.put_release_ensured_gil()
                  code.putln("}")
                  code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname))
                  code.putln("if (this != &__Pyx_other) {")
                  code.put_ensure_gil()
                  for attr in scope.var_entries:
                      if not attr.type.is_cfunction:
1137
                          code.put_var_xdecref(attr, nanny=False)
1138 1139 1140 1141 1142 1143 1144 1145 1146
                          code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname))
                          code.put_var_incref(attr, nanny=False)
                  code.put_release_ensured_gil()
                  code.putln("}")
                  code.putln("return *this;")
                  code.putln("}")
                else:
                  code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname))
                  code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname))
gsamain's avatar
gsamain committed
1147 1148 1149 1150
            if type.is_cyp_class:
                code.putln("// Auto generating default constructor to have Python-like behaviour")
                code.putln("%s(){}" % type.cname)
                alloc_entry = scope.lookup_here("<alloc>")
1151
                if alloc_entry.is_default:
1152
                    code.putln("// Generating default __alloc__ function (used for __new__ calls)")
1153 1154 1155
                    code.putln("static %s { return new %s(); }" %
                               (alloc_entry.type.declaration_code(alloc_entry.cname),
                               type.declaration_code("", deref=1)))
1156 1157
            code.putln("};")

1158 1159 1160 1161
        if type.is_cyp_class:
            code.globalstate.use_utility_code(
                UtilityCode.load("CyObjects", "CyObjects.cpp", proto_block="utility_code_proto_before_types"))

1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
    def generate_cyp_class_mro_method_resolution(self, scope, code):
        """
            Generate overriding methods in derived cypclasses to forward calls to the correct method according
            to the MRO, regardless of the type of the pointer to the object through which the call is made.
            In other words: emulate Python MRO lookup rules using only C++ virtual methods.
        """
        inherited_methods = [
            e for entry in scope.entries.values() for e in entry.all_alternatives()
            if e.is_cfunction
            and e.from_type
            and e.mro_index > 0
1173
            and e.from_type.is_cyp_class    # avoid dealing with methods inherited from non-cypclass bases for now
1174
            and e.name not in ("<init>", "<del>")
1175 1176
            and (not e.type.is_static_method
                 or e.static_cname is not None) # mro-resolve the virtual methods used to dispatch static methods
1177 1178 1179 1180 1181 1182 1183 1184
            and not e.type.has_varargs # avoid dealing with varargs for now (is this ever required anyway ?)
        ]
        if inherited_methods:
            code.putln("")
            code.putln("/* make all inherited (non overriden) methods resolve correctly according to the MRO */")
        for e in inherited_methods:
            modifiers = code.build_function_modifiers(e.func_modifiers)

1185 1186
            arg_names = ["%s_%d" % (arg.cname, i) for i, arg in enumerate(e.type.args)]
            arg_decls = [arg.type.declaration_code(arg_name) for arg, arg_name in zip(e.type.args, arg_names)]
1187 1188 1189 1190 1191
            if e.type.optional_arg_count:
                opt_name = Naming.optional_args_cname
                arg_decls.append(e.type.op_arg_struct.declaration_code(opt_name))
                arg_names.append(opt_name)

1192 1193 1194
            cname = e.cname if not e.type.is_static_method else e.static_cname

            header = e.type.function_header_code(cname, ", ".join(arg_decls))
1195 1196 1197 1198 1199 1200
            if not e.name.startswith("operator "):
                header = e.type.return_type.declaration_code(header)

            return_code = "" if e.type.return_type.is_void else "return "
            resolution = e.from_type.empty_declaration_code()

1201
            body = "%s%s::%s(%s);" % (return_code, resolution, cname, ", ".join(arg_names))
1202 1203 1204 1205 1206

            code.putln("virtual %s%s {%s}" % (modifiers, header, body))
        if inherited_methods:
            code.putln("")

1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
    def generate_cyp_class_static_method_resolution(self, static_method, code):
        """
            Generate a virtual method in cypclass that just forward calls to associated static method.
            The virtual version will serve to correctly dispatch static methods.
        """
        func_type = static_method.type
        modifiers = code.build_function_modifiers(static_method.func_modifiers)

        arg_names = ["%s_%d" % (arg.cname, i) for i, arg in enumerate(func_type.args)]
        arg_decls = [arg.type.declaration_code(arg_name) for arg, arg_name in zip(func_type.args, arg_names)]
        if func_type.optional_arg_count:
            opt_name = Naming.optional_args_cname
            arg_decls.append(func_type.op_arg_struct.declaration_code(opt_name))
            arg_names.append(opt_name)

1222
        header = func_type.function_header_code(static_method.static_cname, ", ".join(arg_decls))
1223 1224 1225 1226 1227
        if not static_method.name.startswith("operator "):
            header = func_type.return_type.declaration_code(header)

        return_code = "" if func_type.return_type.is_void else "return "

1228
        body = "%s%s(%s);" % (return_code, static_method.cname, ", ".join(arg_names))
1229 1230 1231 1232

        code.putln("virtual %s%s {%s}" % (modifiers, header, body))


1233
    def generate_enum_definition(self, entry, code):
1234
        code.mark_pos(entry.pos)
1235 1236
        type = entry.type
        name = entry.cname or entry.name or ""
1237
        header, footer = self.sue_header_footer(type, "enum", name)
1238 1239 1240
        code.putln(header)
        enum_values = entry.enum_values
        if not enum_values:
1241
            error(entry.pos, "Empty enum definition not allowed outside a 'cdef extern from' block")
1242 1243
        else:
            last_entry = enum_values[-1]
1244
            # this does not really generate code, just builds the result value
1245
            for value_entry in enum_values:
1246 1247 1248 1249 1250
                if value_entry.value_node is not None:
                    value_entry.value_node.generate_evaluation_code(code)

            for value_entry in enum_values:
                if value_entry.value_node is None:
1251 1252 1253 1254
                    value_code = value_entry.cname
                else:
                    value_code = ("%s = %s" % (
                        value_entry.cname,
1255
                        value_entry.value_node.result()))
1256 1257 1258 1259
                if value_entry is not last_entry:
                    value_code += ","
                code.putln(value_code)
        code.putln(footer)
1260 1261 1262
        if entry.type.typedef_flag:
            # Not pre-declared.
            code.putln("typedef enum %s %s;" % (name, name))
1263

1264
    def generate_typeobj_predeclaration(self, entry, code):
1265 1266 1267 1268
        code.putln("")
        name = entry.type.typeobj_cname
        if name:
            if entry.visibility == 'extern' and not entry.in_cinclude:
1269
                code.putln("%s %s %s;" % (
1270
                    Naming.extern_c_macro,
1271
                    PyrexTypes.public_decl("PyTypeObject", "DL_IMPORT"),
1272 1273
                    name))
            elif entry.visibility == 'public':
1274
                code.putln("%s %s %s;" % (
1275
                    Naming.extern_c_macro,
1276
                    PyrexTypes.public_decl("PyTypeObject", "DL_EXPORT"),
1277 1278 1279
                    name))
            # ??? Do we really need the rest of this? ???
            #else:
1280
            #    code.putln("static PyTypeObject %s;" % name)
1281

1282
    def generate_exttype_vtable_struct(self, entry, code):
1283 1284 1285
        if not entry.used:
            return

1286
        code.mark_pos(entry.pos)
1287 1288 1289
        # Generate struct declaration for an extension type's vtable.
        type = entry.type
        scope = type.scope
1290 1291 1292

        self.specialize_fused_types(scope)

1293 1294
        if type.vtabstruct_cname:
            code.putln("")
1295
            code.putln("struct %s {" % type.vtabstruct_cname)
1296 1297 1298 1299 1300 1301
            if type.base_type and type.base_type.vtabstruct_cname:
                code.putln("struct %s %s;" % (
                    type.base_type.vtabstruct_cname,
                    Naming.obj_base_cname))
            for method_entry in scope.cfunc_entries:
                if not method_entry.is_inherited:
1302 1303
                    code.putln("%s;" % method_entry.type.declaration_code("(*%s)" % method_entry.cname))
            code.putln("};")
1304

1305
    def generate_exttype_vtabptr_declaration(self, entry, code):
1306 1307 1308
        if not entry.used:
            return

1309
        code.mark_pos(entry.pos)
1310 1311 1312 1313 1314 1315
        # Generate declaration of pointer to an extension type's vtable.
        type = entry.type
        if type.vtabptr_cname:
            code.putln("static struct %s *%s;" % (
                type.vtabstruct_cname,
                type.vtabptr_cname))
1316

1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
    def generate_exttype_final_methods_declaration(self, entry, code):
        if not entry.used:
            return

        code.mark_pos(entry.pos)
        # Generate final methods prototypes
        type = entry.type
        for method_entry in entry.type.scope.cfunc_entries:
            if not method_entry.is_inherited and method_entry.final_func_cname:
                declaration = method_entry.type.declaration_code(
                    method_entry.final_func_cname)
1328 1329
                modifiers = code.build_function_modifiers(method_entry.func_modifiers)
                code.putln("static %s%s;" % (modifiers, declaration))
1330

1331 1332 1333 1334
    def generate_objstruct_predeclaration(self, type, code):
        if not type.scope:
            return
        code.putln(self.sue_predeclaration(type, "struct", type.objstruct_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1335

1336
    def generate_objstruct_definition(self, type, code):
1337 1338 1339 1340
        if type.is_cyp_wrapper:
            # cclass wrappers for cypclass already have an objstruct
            return

1341
        code.mark_pos(type.pos)
1342 1343 1344 1345 1346 1347 1348 1349 1350
        # Generate object struct definition for an
        # extension type.
        if not type.scope:
            return # Forward declared but never defined
        header, footer = \
            self.sue_header_footer(type, "struct", type.objstruct_cname)
        code.putln(header)
        base_type = type.base_type
        if base_type:
1351 1352 1353 1354
            basestruct_cname = base_type.objstruct_cname
            if basestruct_cname == "PyTypeObject":
                # User-defined subclasses of type are heap allocated.
                basestruct_cname = "PyHeapTypeObject"
1355 1356 1357
            code.putln(
                "%s%s %s;" % (
                    ("struct ", "")[base_type.typedef_flag],
1358
                    basestruct_cname,
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
                    Naming.obj_base_cname))
        else:
            code.putln(
                "PyObject_HEAD")
        if type.vtabslot_cname and not (type.base_type and type.base_type.vtabslot_cname):
            code.putln(
                "struct %s *%s;" % (
                    type.vtabstruct_cname,
                    type.vtabslot_cname))
        for attr in type.scope.var_entries:
1369 1370 1371 1372
            if attr.is_declared_generic:
                attr_type = py_object_type
            else:
                attr_type = attr.type
1373
            code.putln(
1374
                "%s;" % attr_type.declaration_code(attr.cname))
1375
        code.putln(footer)
1376 1377 1378
        if type.objtypedef_cname is not None:
            # Only for exposing public typedef name.
            code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname))
1379

1380 1381 1382 1383 1384 1385
    def generate_c_class_declarations(self, env, code, definition, globalstate):
        module_state = globalstate['module_state']
        module_state_defines = globalstate['module_state_defines']
        module_state_clear = globalstate['module_state_clear']
        module_state_traverse = globalstate['module_state_traverse']
        code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
1386
        for entry in env.c_class_entries:
1387
            if definition or entry.defined_in_pxd:
1388 1389
                code.putln("static PyTypeObject *%s = 0;" % (
                    entry.type.typeptr_cname))
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
                module_state.putln("PyTypeObject *%s;" % entry.type.typeptr_cname)
                module_state_defines.putln("#define %s %s->%s" % (
                    entry.type.typeptr_cname,
                    Naming.modulestateglobal_cname,
                    entry.type.typeptr_cname))
                module_state_clear.putln(
                    "Py_CLEAR(clear_module_state->%s);" %
                    entry.type.typeptr_cname)
                module_state_traverse.putln(
                    "Py_VISIT(traverse_module_state->%s);" %
                    entry.type.typeptr_cname)
1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
                if entry.type.typeobj_cname is not None:
                    module_state.putln("PyObject *%s;" % entry.type.typeobj_cname)
                    module_state_defines.putln("#define %s %s->%s" % (
                        entry.type.typeobj_cname,
                        Naming.modulestateglobal_cname,
                        entry.type.typeobj_cname))
                    module_state_clear.putln(
                        "Py_CLEAR(clear_module_state->%s);" % (
                        entry.type.typeobj_cname))
                    module_state_traverse.putln(
                        "Py_VISIT(traverse_module_state->%s);" % (
                        entry.type.typeobj_cname))
1413
        code.putln("#endif")
1414

1415
    def generate_cvariable_declarations(self, env, code, definition):
1416 1417
        if env.is_cython_builtin:
            return
1418 1419
        for entry in env.var_entries:
            if (entry.in_cinclude or entry.in_closure or
1420
                    (entry.visibility == 'private' and not (entry.defined_in_pxd or entry.used))):
1421
                continue
1422 1423 1424 1425 1426 1427 1428 1429 1430

            storage_class = None
            dll_linkage = None
            init = None

            if entry.visibility == 'extern':
                storage_class = Naming.extern_c_macro
                dll_linkage = "DL_IMPORT"
            elif entry.visibility == 'public':
1431
                storage_class = Naming.extern_c_macro
1432 1433 1434 1435 1436 1437
                if definition:
                    dll_linkage = "DL_EXPORT"
                else:
                    dll_linkage = "DL_IMPORT"
            elif entry.visibility == 'private':
                storage_class = "static"
1438 1439
                dll_linkage = None
                if entry.init is not None:
1440
                    init = entry.type.literal_code(entry.init)
1441 1442
            type = entry.type
            cname = entry.cname
1443 1444 1445 1446

            if entry.defined_in_pxd and not definition:
                storage_class = "static"
                dll_linkage = None
1447
                type = CPtrType(type)
1448 1449 1450 1451 1452 1453
                cname = env.mangle(Naming.varptr_prefix, entry.name)
                init = 0

            if storage_class:
                code.put("%s " % storage_class)
            code.put(type.declaration_code(
1454
                cname, dll_linkage=dll_linkage))
1455 1456 1457 1458 1459 1460 1461
            if init is not None:
                code.put_safe(" = %s" % init)
            code.putln(";")
            if entry.cname != cname:
                code.putln("#define %s (*%s)" % (entry.cname, cname))

    def generate_cfunction_declarations(self, env, code, definition):
1462
        for entry in env.cfunc_entries:
1463
            if entry.used or (entry.visibility == 'public' or entry.api):
1464
                generate_cfunction_declaration(entry, env, code, definition)
1465

1466 1467
    def generate_variable_definitions(self, env, code):
        for entry in env.var_entries:
1468
            if not entry.in_cinclude and entry.visibility == "public":
1469 1470
                code.put(entry.type.declaration_code(entry.cname))
                if entry.init is not None:
1471
                    init = entry.type.literal_code(entry.init)
1472 1473 1474
                    code.put_safe(" = %s" % init)
                code.putln(";")

1475 1476 1477 1478 1479
    def generate_typeobj_definitions(self, env, code):
        full_module_name = env.qualified_name
        for entry in env.c_class_entries:
            #print "generate_typeobj_definitions:", entry.name
            #print "...visibility =", entry.visibility
Stefan Behnel's avatar
Stefan Behnel committed
1480
            if entry.visibility != 'extern':
1481 1482 1483 1484
                type = entry.type
                scope = type.scope
                if scope: # could be None if there was an error
                    self.generate_exttype_vtable(scope, code)
1485
                    self.generate_new_function(scope, code, entry)
1486
                    self.generate_dealloc_function(scope, code)
1487
                    if scope.needs_gc():
1488
                        self.generate_traverse_function(scope, code, entry)
1489 1490
                        if scope.needs_tp_clear():
                            self.generate_clear_function(scope, code, entry)
1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
                    if scope.defines_any_special(["__getitem__"]):
                        self.generate_getitem_int_function(scope, code)
                    if scope.defines_any_special(["__setitem__", "__delitem__"]):
                        self.generate_ass_subscript_function(scope, code)
                    if scope.defines_any_special(["__getslice__", "__setslice__", "__delslice__"]):
                        warning(self.pos,
                                "__getslice__, __setslice__, and __delslice__ are not supported by Python 3, "
                                "use __getitem__, __setitem__, and __delitem__ instead", 1)
                        code.putln("#if PY_MAJOR_VERSION >= 3")
                        code.putln("#error __getslice__, __setslice__, and __delslice__ not supported in Python 3.")
                        code.putln("#endif")
                    if scope.defines_any_special(["__setslice__", "__delslice__"]):
                        self.generate_ass_slice_function(scope, code)
                    if scope.defines_any_special(["__getattr__", "__getattribute__"]):
                        self.generate_getattro_function(scope, code)
                    if scope.defines_any_special(["__setattr__", "__delattr__"]):
                        self.generate_setattro_function(scope, code)
                    if scope.defines_any_special(["__get__"]):
                        self.generate_descr_get_function(scope, code)
                    if scope.defines_any_special(["__set__", "__delete__"]):
                        self.generate_descr_set_function(scope, code)
                    if not scope.is_closure_class_scope and scope.defines_any(["__dict__"]):
                        self.generate_dict_getter_function(scope, code)
                    if scope.defines_any_special(TypeSlots.richcmp_special_methods):
                        self.generate_richcmp_function(scope, code)
1516 1517 1518
                    self.generate_property_accessors(scope, code)
                    self.generate_method_table(scope, code)
                    self.generate_getset_table(scope, code)
1519
                    code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
1520
                    self.generate_typeobj_spec(entry, code)
1521
                    code.putln("#else")
1522
                    self.generate_typeobj_definition(full_module_name, entry, code)
1523
                    code.putln("#endif")
1524

1525 1526 1527 1528 1529 1530 1531
    def generate_exttype_vtable(self, scope, code):
        # Generate the definition of an extension type's vtable.
        type = scope.parent_type
        if type.vtable_cname:
            code.putln("static struct %s %s;" % (
                type.vtabstruct_cname,
                type.vtable_cname))
1532

1533 1534 1535 1536 1537
    def generate_self_cast(self, scope, code):
        type = scope.parent_type
        code.putln(
            "%s = (%s)o;" % (
                type.declaration_code("p"),
1538
                type.empty_declaration_code()))
1539

1540
    def generate_new_function(self, scope, code, cclass_entry):
1541
        tp_slot = TypeSlots.ConstructorSlot("tp_new", "__cinit__")
1542
        slot_func = scope.mangle_internal("tp_new")
1543 1544 1545
        if tp_slot.slot_code(scope) != slot_func:
            return  # never used

1546 1547
        type = scope.parent_type
        base_type = type.base_type
1548

1549
        have_entries, (py_attrs, py_buffers, memoryview_slices) = \
1550
                        scope.get_refcounted_entries()
1551
        is_final_type = scope.parent_type.is_final_type
1552 1553 1554
        if scope.is_internal:
            # internal classes (should) never need None inits, normal zeroing will do
            py_attrs = []
1555

1556
        # unlike normal cpp_class attributes, cyp_class attributes are always held as pointers
1557
        cpp_class_attrs = [entry for entry in scope.var_entries
1558
                           if entry.type.is_cpp_class and not entry.type.is_cyp_class]
1559

1560 1561 1562
        cyp_class_attrs = [entry for entry in scope.var_entries
                           if entry.type.is_cyp_class]

1563 1564 1565 1566 1567
        cinit_func_entry = scope.lookup_here("__cinit__")
        if cinit_func_entry and not cinit_func_entry.is_special:
            cinit_func_entry = None

        if base_type or (cinit_func_entry and not cinit_func_entry.trivial_signature):
1568 1569 1570 1571
            unused_marker = ''
        else:
            unused_marker = 'CYTHON_UNUSED '

1572 1573 1574 1575 1576 1577 1578
        if base_type:
            freelist_size = 0  # not currently supported
        else:
            freelist_size = scope.directives.get('freelist', 0)
        freelist_name = scope.mangle_internal(Naming.freelist_name)
        freecount_name = scope.mangle_internal(Naming.freecount_name)

1579 1580 1581
        decls = code.globalstate['decls']
        decls.putln("static PyObject *%s(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/" %
                    slot_func)
1582
        code.putln("")
1583 1584 1585 1586 1587 1588
        if freelist_size:
            code.putln("static %s[%d];" % (
                scope.parent_type.declaration_code(freelist_name),
                freelist_size))
            code.putln("static int %s = 0;" % freecount_name)
            code.putln("")
1589
        code.putln(
1590 1591
            "static PyObject *%s(PyTypeObject *t, %sPyObject *a, %sPyObject *k) {" % (
                slot_func, unused_marker, unused_marker))
1592

1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
        # for cyp wrappers, just allocate the cyobject and return the wrapper
        # let the wrapped __init__ handle initialisation
        if type.is_cyp_wrapper:
            from .CypclassWrapper import generate_cypclass_wrapper_allocation

            code.putln("if (t != %s) {" % type.typeptr_cname)
            code.putln(
                "PyErr_SetString(PyExc_TypeError,\"Cannot build a subtype of %s with %s.__new__\");"
                % (scope.qualified_name, scope.qualified_name)
            )
            code.putln("return NULL;")
            code.putln("}")
            code.putln("CyObject * self = %s();" % type.wrapped_alloc)
            generate_cypclass_wrapper_allocation(code, type)
            code.putln(
                "PyObject* wrapper = reinterpret_cast<PyObject *>(static_cast<%s *>(self));"
                % Naming.cypclass_wrapper_layout_type
            )
            code.putln("Py_INCREF(wrapper);")
            code.putln("return wrapper;")
            code.putln("}")
            return

1616 1617
        need_self_cast = (type.vtabslot_cname or
                          (py_buffers or memoryview_slices or py_attrs) or
1618
                          cpp_class_attrs or cyp_class_attrs)
1619
        if need_self_cast:
1620
            code.putln("%s;" % scope.parent_type.declaration_code("p"))
1621
        if base_type:
1622 1623 1624 1625 1626
            code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
            code.putln("newfunc new_func = (newfunc)PyType_GetSlot(%s, Py_tp_new);" %
                base_type.typeptr_cname)
            code.putln("PyObject *o = new_func(t, a, k);")
            code.putln("#else")
Robert Bradshaw's avatar
Robert Bradshaw committed
1627 1628
            tp_new = TypeSlots.get_base_slot_function(scope, tp_slot)
            if tp_new is None:
1629
                tp_new = "%s->tp_new" % base_type.typeptr_cname
1630
            code.putln("PyObject *o = %s(t, a, k);" % tp_new)
1631
            code.putln("#endif")
1632
        else:
1633
            code.putln("PyObject *o;")
1634 1635 1636 1637
            code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
            code.putln("allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);")
            code.putln("o = alloc_func(t, 0);")
            code.putln("#else")
1638
            if freelist_size:
1639 1640
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("IncludeStringH", "StringTools.c"))
1641
                if is_final_type:
1642
                    type_safety_check = ''
1643
                else:
1644
                    type_safety_check = ' & ((t->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)'
1645
                obj_struct = type.declaration_code("", deref=True)
1646 1647 1648
                code.putln(
                    "if (CYTHON_COMPILING_IN_CPYTHON && likely((%s > 0) & (t->tp_basicsize == sizeof(%s))%s)) {" % (
                        freecount_name, obj_struct, type_safety_check))
1649 1650
                code.putln("o = (PyObject*)%s[--%s];" % (
                    freelist_name, freecount_name))
1651
                code.putln("memset(o, 0, sizeof(%s));" % obj_struct)
1652
                code.putln("(void) PyObject_INIT(o, t);")
1653 1654 1655
                if scope.needs_gc():
                    code.putln("PyObject_GC_Track(o);")
                code.putln("} else {")
1656 1657 1658 1659 1660 1661 1662
            if not is_final_type:
                code.putln("if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {")
            code.putln("o = (*t->tp_alloc)(t, 0);")
            if not is_final_type:
                code.putln("} else {")
                code.putln("o = (PyObject *) PyBaseObject_Type.tp_new(t, %s, 0);" % Naming.empty_tuple)
                code.putln("}")
1663
        code.putln("if (unlikely(!o)) return 0;")
1664 1665
        if freelist_size and not base_type:
            code.putln('}')
1666 1667
        if not base_type:
            code.putln("#endif")
1668
        if need_self_cast:
1669
            code.putln("p = %s;" % type.cast_code("o"))
1670
        #if need_self_cast:
Robert Bradshaw's avatar
Robert Bradshaw committed
1671
        #    self.generate_self_cast(scope, code)
1672 1673 1674 1675

        # from this point on, ensure DECREF(o) on failure
        needs_error_cleanup = False

1676
        if type.vtabslot_cname:
1677 1678 1679 1680 1681
            vtab_base_type = type
            while vtab_base_type.base_type and vtab_base_type.base_type.vtabstruct_cname:
                vtab_base_type = vtab_base_type.base_type
            if vtab_base_type is not type:
                struct_type_cast = "(struct %s*)" % vtab_base_type.vtabstruct_cname
1682 1683 1684
            else:
                struct_type_cast = ""
            code.putln("p->%s = %s%s;" % (
1685
                type.vtabslot_cname,
1686
                struct_type_cast, type.vtabptr_cname))
1687

Robert Bradshaw's avatar
Robert Bradshaw committed
1688
        for entry in cpp_class_attrs:
Stefan Behnel's avatar
Stefan Behnel committed
1689 1690
            code.putln("new((void*)&(p->%s)) %s();" % (
                entry.cname, entry.type.empty_declaration_code()))
1691

1692 1693 1694
        for entry in cyp_class_attrs:
            code.putln("p->%s = NULL;" % entry.cname)

1695
        for entry in py_attrs:
1696
            if entry.name == "__dict__":
1697 1698 1699
                needs_error_cleanup = True
                code.put("p->%s = PyDict_New(); if (unlikely(!p->%s)) goto bad;" % (
                    entry.cname, entry.cname))
1700 1701
            else:
                code.put_init_var_to_py_none(entry, "p->%s", nanny=False)
1702

1703
        for entry in memoryview_slices:
1704
            code.putln("p->%s.data = NULL;" % entry.cname)
1705
            code.putln("p->%s.memview = NULL;" % entry.cname)
1706 1707 1708 1709 1710 1711 1712

        for entry in py_buffers:
            code.putln("p->%s.obj = NULL;" % entry.cname)

        if cclass_entry.cname == '__pyx_memoryviewslice':
            code.putln("p->from_slice.memview = NULL;")

1713 1714
        if cinit_func_entry:
            if cinit_func_entry.trivial_signature:
1715 1716 1717
                cinit_args = "o, %s, NULL" % Naming.empty_tuple
            else:
                cinit_args = "o, a, k"
1718 1719
            needs_error_cleanup = True
            code.putln("if (unlikely(%s(%s) < 0)) goto bad;" % (
1720
                cinit_func_entry.func_cname, cinit_args))
1721

1722 1723
        code.putln(
            "return o;")
1724 1725 1726 1727
        if needs_error_cleanup:
            code.putln("bad:")
            code.put_decref_clear("o", py_object_type, nanny=False)
            code.putln("return NULL;")
1728 1729
        code.putln(
            "}")
1730

1731
    def generate_dealloc_function(self, scope, code):
1732
        tp_slot = TypeSlots.ConstructorSlot("tp_dealloc", '__dealloc__')
1733
        slot_func = scope.mangle_internal("tp_dealloc")
1734
        base_type = scope.parent_type.base_type
1735
        if tp_slot.slot_code(scope) != slot_func:
1736
            return  # never used
1737 1738

        slot_func_cname = scope.mangle_internal("tp_dealloc")
1739
        code.putln("")
1740 1741 1742 1743 1744
        cdealloc_func_entry = scope.lookup_here("__dealloc__")
        if cdealloc_func_entry and not cdealloc_func_entry.is_special:
            cdealloc_func_entry = None
        if cdealloc_func_entry is None:
            code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
1745
        code.putln(
1746
            "static void %s(PyObject *o) {" % slot_func_cname)
1747

1748
        # for cyp wrappers, just decrement the atomic counter of the underlying type
1749 1750
        parent_type = scope.parent_type
        if parent_type.is_cyp_wrapper:
1751
            self.generate_self_cast(scope, code)
1752
            code.putln(
1753
                "CyObject * p_nogil_cyobject = static_cast<CyObject *>(p);"
1754 1755
            )
            code.putln("Cy_DECREF(p_nogil_cyobject);")
1756 1757 1758 1759 1760
            code.putln("}")
            if cdealloc_func_entry is None:
                code.putln("#endif")
            return

1761 1762
        is_final_type = scope.parent_type.is_final_type
        needs_gc = scope.needs_gc()
1763
        needs_trashcan = scope.needs_trashcan()
1764

1765
        weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
1766 1767 1768
        if weakref_slot not in scope.var_entries:
            weakref_slot = None

1769
        dict_slot = scope.lookup_here("__dict__") if not scope.is_closure_class_scope else None
1770 1771 1772
        if dict_slot not in scope.var_entries:
            dict_slot = None

1773
        _, (py_attrs, _, memoryview_slices) = scope.get_refcounted_entries()
1774

1775
        # unlike normal cpp_class attributes, cyp_class attributes are always held as pointers
1776
        cpp_class_attrs = [entry for entry in scope.var_entries
1777
                           if entry.type.is_cpp_class and not entry.type.is_cyp_class]
1778

1779 1780 1781 1782
        cyp_class_attrs = [entry for entry in scope.var_entries
                           if entry.type.is_cyp_class]

        if py_attrs or cpp_class_attrs or cyp_class_attrs or memoryview_slices or weakref_slot or dict_slot:
1783
            self.generate_self_cast(scope, code)
1784 1785 1786

        if not is_final_type:
            # in Py3.4+, call tp_finalize() as early as possible
1787
            code.putln("#if CYTHON_USE_TP_FINALIZE")
1788 1789 1790 1791 1792
            if needs_gc:
                finalised_check = '!_PyGC_FINALIZED(o)'
            else:
                finalised_check = (
                    '(!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))')
1793
            code.putln(
1794 1795
                "if (unlikely("
                "(PY_VERSION_HEX >= 0x03080000 || PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE))"
1796
                " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check)
1797 1798 1799 1800 1801 1802 1803 1804 1805
            # if instance was resurrected by finaliser, return
            code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;")
            code.putln("}")
            code.putln("#endif")

        if needs_gc:
            # We must mark this object as (gc) untracked while tearing
            # it down, lest the garbage collection is invoked while
            # running this destructor.
1806
            code.putln("PyObject_GC_UnTrack(o);")
1807

1808 1809 1810 1811 1812
        if needs_trashcan:
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c"))
            code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname)

1813
        if weakref_slot:
1814 1815 1816 1817
            # We must clean the weakreferences before calling the user's __dealloc__
            # because if the __dealloc__ releases the GIL, a weakref can be
            # dereferenced accessing the object in an inconsistent state or
            # resurrecting it.
1818
            code.putln("if (p->__weakref__) PyObject_ClearWeakRefs(o);")
1819

1820 1821 1822
        # call the user's __dealloc__
        self.generate_usr_dealloc_call(scope, code)

1823 1824 1825
        if dict_slot:
            code.putln("if (p->__dict__) PyDict_Clear(p->__dict__);")

Robert Bradshaw's avatar
Robert Bradshaw committed
1826
        for entry in cpp_class_attrs:
1827
            code.putln("__Pyx_call_destructor(p->%s);" % entry.cname)
1828

1829 1830 1831
        for entry in cyp_class_attrs:
            code.putln("Cy_XDECREF(p->%s);" % entry.cname)

1832
        for entry in (py_attrs + memoryview_slices):
1833
            code.put_xdecref_clear("p->%s" % entry.cname, entry.type, nanny=False,
1834
                                   clear_before_decref=True, have_gil=True)
1835

1836
        if base_type:
1837 1838 1839
            if needs_gc:
                # The base class deallocator probably expects this to be tracked,
                # so undo the untracking above.
1840 1841 1842
                if base_type.scope and base_type.scope.needs_gc():
                    code.putln("PyObject_GC_Track(o);")
                else:
1843
                    code.putln("#if CYTHON_USE_TYPE_SLOTS")
1844 1845 1846
                    code.putln("if (PyType_IS_GC(Py_TYPE(o)->tp_base))")
                    code.putln("#endif")
                    code.putln("PyObject_GC_Track(o);")
1847

Robert Bradshaw's avatar
Robert Bradshaw committed
1848
            tp_dealloc = TypeSlots.get_base_slot_function(scope, tp_slot)
1849 1850
            if tp_dealloc is not None:
                code.putln("%s(o);" % tp_dealloc)
1851 1852
            elif base_type.is_builtin_type:
                code.putln("%s->tp_dealloc(o);" % base_type.typeptr_cname)
1853 1854 1855 1856 1857 1858
            else:
                # This is an externally defined type.  Calling through the
                # cimported base type pointer directly interacts badly with
                # the module cleanup, which may already have cleared it.
                # In that case, fall back to traversing the type hierarchy.
                base_cname = base_type.typeptr_cname
1859 1860 1861
                code.putln("if (likely(%s)) %s->tp_dealloc(o); "
                           "else __Pyx_call_next_tp_dealloc(o, %s);" % (
                               base_cname, base_cname, slot_func_cname))
1862 1863
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("CallNextTpDealloc", "ExtensionTypes.c"))
1864
        else:
1865 1866 1867 1868 1869
            freelist_size = scope.directives.get('freelist', 0)
            if freelist_size:
                freelist_name = scope.mangle_internal(Naming.freelist_name)
                freecount_name = scope.mangle_internal(Naming.freecount_name)

1870 1871 1872 1873 1874 1875
                if is_final_type:
                    type_safety_check = ''
                else:
                    type_safety_check = (
                        ' & ((Py_TYPE(o)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)')

1876
                type = scope.parent_type
1877 1878 1879 1880 1881 1882
                code.putln(
                    "if (CYTHON_COMPILING_IN_CPYTHON && ((%s < %d) & (Py_TYPE(o)->tp_basicsize == sizeof(%s))%s)) {" % (
                        freecount_name,
                        freelist_size,
                        type.declaration_code("", deref=True),
                        type_safety_check))
1883 1884 1885 1886 1887 1888
                code.putln("%s[%s++] = %s;" % (
                    freelist_name, freecount_name, type.cast_code("o")))
                code.putln("} else {")
            code.putln("(*Py_TYPE(o)->tp_free)(o);")
            if freelist_size:
                code.putln("}")
1889 1890 1891 1892

        if needs_trashcan:
            code.putln("__Pyx_TRASHCAN_END")

1893 1894
        code.putln(
            "}")
1895 1896
        if cdealloc_func_entry is None:
            code.putln("#endif")
1897

1898 1899
    def generate_usr_dealloc_call(self, scope, code):
        entry = scope.lookup_here("__dealloc__")
1900
        if not entry or not entry.is_special:
1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
            return

        code.putln("{")
        code.putln("PyObject *etype, *eval, *etb;")
        code.putln("PyErr_Fetch(&etype, &eval, &etb);")
        code.putln("++Py_REFCNT(o);")
        code.putln("%s(o);" % entry.func_cname)
        code.putln("--Py_REFCNT(o);")
        code.putln("PyErr_Restore(etype, eval, etb);")
        code.putln("}")
1911

1912
    def generate_traverse_function(self, scope, code, cclass_entry):
1913 1914
        tp_slot = TypeSlots.GCDependentSlot("tp_traverse")
        slot_func = scope.mangle_internal("tp_traverse")
1915
        base_type = scope.parent_type.base_type
1916
        if tp_slot.slot_code(scope) != slot_func:
1917
            return  # never used
1918 1919
        code.putln("")
        code.putln(
1920
            "static int %s(PyObject *o, visitproc v, void *a) {" % slot_func)
1921

1922 1923
        have_entries, (py_attrs, py_buffers, memoryview_slices) = (
            scope.get_refcounted_entries(include_gc_simple=False))
1924

1925
        if base_type or py_attrs:
1926
            code.putln("int e;")
1927 1928

        if py_attrs or py_buffers:
1929
            self.generate_self_cast(scope, code)
1930

1931
        if base_type:
1932
            # want to call it explicitly if possible so inlining can be performed
Robert Bradshaw's avatar
Robert Bradshaw committed
1933 1934 1935
            static_call = TypeSlots.get_base_slot_function(scope, tp_slot)
            if static_call:
                code.putln("e = %s(o, v, a); if (e) return e;" % static_call)
1936 1937 1938 1939
            elif base_type.is_builtin_type:
                base_cname = base_type.typeptr_cname
                code.putln("if (!%s->tp_traverse); else { e = %s->tp_traverse(o,v,a); if (e) return e; }" % (
                    base_cname, base_cname))
1940
            else:
1941 1942 1943 1944 1945
                # This is an externally defined type.  Calling through the
                # cimported base type pointer directly interacts badly with
                # the module cleanup, which may already have cleared it.
                # In that case, fall back to traversing the type hierarchy.
                base_cname = base_type.typeptr_cname
1946 1947 1948 1949
                code.putln(
                    "e = ((likely(%s)) ? ((%s->tp_traverse) ? %s->tp_traverse(o, v, a) : 0) : "
                    "__Pyx_call_next_tp_traverse(o, v, a, %s)); if (e) return e;" % (
                        base_cname, base_cname, base_cname, slot_func))
1950 1951
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("CallNextTpTraverse", "ExtensionTypes.c"))
1952

1953 1954
        for entry in py_attrs:
            var_code = "p->%s" % entry.cname
1955
            var_as_pyobject = PyrexTypes.typecast(py_object_type, entry.type, var_code)
1956
            code.putln("if (%s) {" % var_code)
1957
            code.putln("e = (*v)(%s, a); if (e) return e;" % var_as_pyobject)
1958
            code.putln("}")
1959

Stefan Behnel's avatar
Stefan Behnel committed
1960 1961
        # Traverse buffer exporting objects.
        # Note: not traversing memoryview attributes of memoryview slices!
1962 1963
        # When triggered by the GC, it would cause multiple visits (gc_refs
        # subtractions which is not matched by its reference count!)
Stefan Behnel's avatar
Stefan Behnel committed
1964 1965
        for entry in py_buffers:
            cname = entry.cname + ".obj"
1966
            code.putln("if (p->%s) {" % cname)
1967
            code.putln("e = (*v)(p->%s, a); if (e) return e;" % cname)
1968 1969
            code.putln("}")

1970 1971
        code.putln("return 0;")
        code.putln("}")
1972

1973
    def generate_clear_function(self, scope, code, cclass_entry):
1974
        tp_slot = TypeSlots.get_slot_by_name("tp_clear")
1975
        slot_func = scope.mangle_internal("tp_clear")
1976
        base_type = scope.parent_type.base_type
1977 1978
        if tp_slot.slot_code(scope) != slot_func:
            return # never used
1979

1980 1981
        have_entries, (py_attrs, py_buffers, memoryview_slices) = (
            scope.get_refcounted_entries(include_gc_simple=False))
1982

1983 1984 1985 1986 1987 1988 1989 1990
        if py_attrs or py_buffers or base_type:
            unused = ''
        else:
            unused = 'CYTHON_UNUSED '

        code.putln("")
        code.putln("static int %s(%sPyObject *o) {" % (slot_func, unused))

1991 1992 1993
        if py_attrs and Options.clear_to_none:
            code.putln("PyObject* tmp;")

1994
        if py_attrs or py_buffers:
1995
            self.generate_self_cast(scope, code)
1996

1997
        if base_type:
1998
            # want to call it explicitly if possible so inlining can be performed
Robert Bradshaw's avatar
Robert Bradshaw committed
1999 2000 2001
            static_call = TypeSlots.get_base_slot_function(scope, tp_slot)
            if static_call:
                code.putln("%s(o);" % static_call)
2002 2003 2004 2005
            elif base_type.is_builtin_type:
                base_cname = base_type.typeptr_cname
                code.putln("if (!%s->tp_clear); else %s->tp_clear(o);" % (
                    base_cname, base_cname))
2006
            else:
2007 2008 2009 2010 2011
                # This is an externally defined type.  Calling through the
                # cimported base type pointer directly interacts badly with
                # the module cleanup, which may already have cleared it.
                # In that case, fall back to traversing the type hierarchy.
                base_cname = base_type.typeptr_cname
2012 2013 2014
                code.putln(
                    "if (likely(%s)) { if (%s->tp_clear) %s->tp_clear(o); } else __Pyx_call_next_tp_clear(o, %s);" % (
                        base_cname, base_cname, base_cname, slot_func))
2015 2016
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("CallNextTpClear", "ExtensionTypes.c"))
2017

2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
        if Options.clear_to_none:
            for entry in py_attrs:
                name = "p->%s" % entry.cname
                code.putln("tmp = ((PyObject*)%s);" % name)
                if entry.is_declared_generic:
                    code.put_init_to_py_none(name, py_object_type, nanny=False)
                else:
                    code.put_init_to_py_none(name, entry.type, nanny=False)
                code.putln("Py_XDECREF(tmp);")
        else:
            for entry in py_attrs:
                code.putln("Py_CLEAR(p->%s);" % entry.cname)
2030 2031

        for entry in py_buffers:
2032
            # Note: shouldn't this call __Pyx_ReleaseBuffer ??
2033 2034 2035 2036 2037
            code.putln("Py_CLEAR(p->%s.obj);" % entry.cname)

        if cclass_entry.cname == '__pyx_memoryviewslice':
            code.putln("__PYX_XDEC_MEMVIEW(&p->from_slice, 1);")

2038 2039
        code.putln("return 0;")
        code.putln("}")
2040

2041 2042 2043 2044 2045
    def generate_getitem_int_function(self, scope, code):
        # This function is put into the sq_item slot when
        # a __getitem__ method is present. It converts its
        # argument to a Python integer and calls mp_subscript.
        code.putln(
2046 2047
            "static PyObject *%s(PyObject *o, Py_ssize_t i) {" % (
                scope.mangle_internal("sq_item")))
2048
        code.putln(
2049
            "PyObject *r;")
2050
        code.putln(
2051
            "PyObject *x = PyInt_FromSsize_t(i); if(!x) return 0;")
2052
        code.putln(
2053
            "r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x);")
2054
        code.putln(
2055
            "Py_DECREF(x);")
2056
        code.putln(
2057
            "return r;")
2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
        code.putln(
            "}")

    def generate_ass_subscript_function(self, scope, code):
        # Setting and deleting an item are both done through
        # the ass_subscript method, so we dispatch to user's __setitem__
        # or __delitem__, or raise an exception.
        base_type = scope.parent_type.base_type
        set_entry = scope.lookup_here("__setitem__")
        del_entry = scope.lookup_here("__delitem__")
        code.putln("")
        code.putln(
2070 2071
            "static int %s(PyObject *o, PyObject *i, PyObject *v) {" % (
                scope.mangle_internal("mp_ass_subscript")))
2072
        code.putln(
2073
            "if (v) {")
2074
        if set_entry:
2075
            code.putln("return %s(o, i, v);" % set_entry.func_cname)
2076 2077 2078 2079
        else:
            self.generate_guarded_basetype_call(
                base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code)
            code.putln(
2080
                "PyErr_Format(PyExc_NotImplementedError,")
2081
            code.putln(
2082
                '  "Subscript assignment not supported by %.200s", Py_TYPE(o)->tp_name);')
2083
            code.putln(
2084
                "return -1;")
2085
        code.putln(
2086
            "}")
2087
        code.putln(
2088
            "else {")
2089 2090
        if del_entry:
            code.putln(
2091 2092
                "return %s(o, i);" % (
                    del_entry.func_cname))
2093 2094 2095 2096
        else:
            self.generate_guarded_basetype_call(
                base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code)
            code.putln(
2097
                "PyErr_Format(PyExc_NotImplementedError,")
2098
            code.putln(
2099
                '  "Subscript deletion not supported by %.200s", Py_TYPE(o)->tp_name);')
2100
            code.putln(
2101
                "return -1;")
2102
        code.putln(
2103
            "}")
2104 2105
        code.putln(
            "}")
2106

2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
    def generate_guarded_basetype_call(
            self, base_type, substructure, slot, args, code):
        if base_type:
            base_tpname = base_type.typeptr_cname
            if substructure:
                code.putln(
                    "if (%s->%s && %s->%s->%s)" % (
                        base_tpname, substructure, base_tpname, substructure, slot))
                code.putln(
                    "  return %s->%s->%s(%s);" % (
                        base_tpname, substructure, slot, args))
            else:
                code.putln(
                    "if (%s->%s)" % (
                        base_tpname, slot))
                code.putln(
                    "  return %s->%s(%s);" % (
                        base_tpname, slot, args))

    def generate_ass_slice_function(self, scope, code):
        # Setting and deleting a slice are both done through
        # the ass_slice method, so we dispatch to user's __setslice__
        # or __delslice__, or raise an exception.
        base_type = scope.parent_type.base_type
        set_entry = scope.lookup_here("__setslice__")
        del_entry = scope.lookup_here("__delslice__")
        code.putln("")
        code.putln(
2135 2136
            "static int %s(PyObject *o, Py_ssize_t i, Py_ssize_t j, PyObject *v) {" % (
                scope.mangle_internal("sq_ass_slice")))
2137
        code.putln(
2138
            "if (v) {")
2139 2140
        if set_entry:
            code.putln(
2141 2142
                "return %s(o, i, j, v);" % (
                    set_entry.func_cname))
2143 2144 2145 2146
        else:
            self.generate_guarded_basetype_call(
                base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code)
            code.putln(
2147
                "PyErr_Format(PyExc_NotImplementedError,")
2148
            code.putln(
2149
                '  "2-element slice assignment not supported by %.200s", Py_TYPE(o)->tp_name);')
2150
            code.putln(
2151
                "return -1;")
2152
        code.putln(
2153
            "}")
2154
        code.putln(
2155
            "else {")
2156 2157
        if del_entry:
            code.putln(
2158 2159
                "return %s(o, i, j);" % (
                    del_entry.func_cname))
2160 2161 2162 2163
        else:
            self.generate_guarded_basetype_call(
                base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code)
            code.putln(
2164
                "PyErr_Format(PyExc_NotImplementedError,")
2165
            code.putln(
2166
                '  "2-element slice deletion not supported by %.200s", Py_TYPE(o)->tp_name);')
2167
            code.putln(
2168
                "return -1;")
2169
        code.putln(
2170
            "}")
2171 2172 2173
        code.putln(
            "}")

2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
    def generate_richcmp_function(self, scope, code):
        if scope.lookup_here("__richcmp__"):
            # user implemented, nothing to do
            return
        # otherwise, we have to generate it from the Python special methods
        richcmp_cfunc = scope.mangle_internal("tp_richcompare")
        code.putln("")
        code.putln("static PyObject *%s(PyObject *o1, PyObject *o2, int op) {" % richcmp_cfunc)
        code.putln("switch (op) {")

        class_scopes = []
        cls = scope.parent_type
        while cls is not None and not cls.entry.visibility == 'extern':
            class_scopes.append(cls.scope)
            cls = cls.scope.parent_type.base_type
        assert scope in class_scopes

        extern_parent = None
        if cls and cls.entry.visibility == 'extern':
            # need to call up into base classes as we may not know all implemented comparison methods
            extern_parent = cls if cls.typeptr_cname else scope.parent_type.base_type

        eq_entry = None
        has_ne = False
        for cmp_method in TypeSlots.richcmp_special_methods:
            for class_scope in class_scopes:
                entry = class_scope.lookup_here(cmp_method)
                if entry is not None:
                    break
            else:
                continue

            cmp_type = cmp_method.strip('_').upper()  # e.g. "__eq__" -> EQ
            code.putln("case Py_%s: {" % cmp_type)
            if cmp_method == '__eq__':
                eq_entry = entry
2210 2211
                # Python itself does not do this optimisation, it seems...
                #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_True);")
2212 2213
            elif cmp_method == '__ne__':
                has_ne = True
2214 2215
                # Python itself does not do this optimisation, it seems...
                #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);")
2216 2217 2218 2219 2220 2221
            code.putln("return %s(o1, o2);" % entry.func_cname)
            code.putln("}")

        if eq_entry and not has_ne and not extern_parent:
            code.putln("case Py_NE: {")
            code.putln("PyObject *ret;")
2222 2223
            # Python itself does not do this optimisation, it seems...
            #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);")
2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243
            code.putln("ret = %s(o1, o2);" % eq_entry.func_cname)
            code.putln("if (likely(ret && ret != Py_NotImplemented)) {")
            code.putln("int b = __Pyx_PyObject_IsTrue(ret); Py_DECREF(ret);")
            code.putln("if (unlikely(b < 0)) return NULL;")
            code.putln("ret = (b) ? Py_False : Py_True;")
            code.putln("Py_INCREF(ret);")
            code.putln("}")
            code.putln("return ret;")
            code.putln("}")

        code.putln("default: {")
        if extern_parent and extern_parent.typeptr_cname:
            code.putln("if (likely(%s->tp_richcompare)) return %s->tp_richcompare(o1, o2, op);" % (
                extern_parent.typeptr_cname, extern_parent.typeptr_cname))
        code.putln("return __Pyx_NewRef(Py_NotImplemented);")
        code.putln("}")

        code.putln("}")  # switch
        code.putln("}")

2244
    def generate_getattro_function(self, scope, code):
2245 2246 2247
        # First try to get the attribute using __getattribute__, if defined, or
        # PyObject_GenericGetAttr.
        #
2248 2249 2250
        # If that raises an AttributeError, call the __getattr__ if defined.
        #
        # In both cases, defined can be in this class, or any base class.
2251
        def lookup_here_or_base(n, tp=None, extern_return=None):
2252
            # Recursive lookup
2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263
            if tp is None:
                tp = scope.parent_type
            r = tp.scope.lookup_here(n)
            if r is None:
                if tp.is_external and extern_return is not None:
                    return extern_return
                if tp.base_type is not None:
                    return lookup_here_or_base(n, tp.base_type)
            return r

        has_instance_dict = lookup_here_or_base("__dict__", extern_return="extern")
2264 2265
        getattr_entry = lookup_here_or_base("__getattr__")
        getattribute_entry = lookup_here_or_base("__getattribute__")
2266 2267
        code.putln("")
        code.putln(
2268 2269
            "static PyObject *%s(PyObject *o, PyObject *n) {" % (
                scope.mangle_internal("tp_getattro")))
2270 2271
        if getattribute_entry is not None:
            code.putln(
2272 2273
                "PyObject *v = %s(o, n);" % (
                    getattribute_entry.func_cname))
2274
        else:
2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
            if not has_instance_dict and scope.parent_type.is_final_type:
                # Final with no dict => use faster type attribute lookup.
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("PyObject_GenericGetAttrNoDict", "ObjectHandling.c"))
                generic_getattr_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict"
            elif not has_instance_dict or has_instance_dict == "extern":
                # No dict in the known ancestors, but don't know about extern ancestors or subtypes.
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("PyObject_GenericGetAttr", "ObjectHandling.c"))
                generic_getattr_cfunc = "__Pyx_PyObject_GenericGetAttr"
            else:
                generic_getattr_cfunc = "PyObject_GenericGetAttr"
2287
            code.putln(
2288
                "PyObject *v = %s(o, n);" % generic_getattr_cfunc)
2289 2290
        if getattr_entry is not None:
            code.putln(
2291
                "if (!v && PyErr_ExceptionMatches(PyExc_AttributeError)) {")
2292 2293 2294
            code.putln(
                "PyErr_Clear();")
            code.putln(
2295 2296
                "v = %s(o, n);" % (
                    getattr_entry.func_cname))
2297
            code.putln(
2298 2299
                "}")
        code.putln(
2300
            "return v;")
2301 2302
        code.putln(
            "}")
2303

2304 2305 2306 2307 2308 2309 2310 2311 2312
    def generate_setattro_function(self, scope, code):
        # Setting and deleting an attribute are both done through
        # the setattro method, so we dispatch to user's __setattr__
        # or __delattr__ or fall back on PyObject_GenericSetAttr.
        base_type = scope.parent_type.base_type
        set_entry = scope.lookup_here("__setattr__")
        del_entry = scope.lookup_here("__delattr__")
        code.putln("")
        code.putln(
2313 2314
            "static int %s(PyObject *o, PyObject *n, PyObject *v) {" % (
                scope.mangle_internal("tp_setattro")))
2315
        code.putln(
2316
            "if (v) {")
2317 2318
        if set_entry:
            code.putln(
2319 2320
                "return %s(o, n, v);" % (
                    set_entry.func_cname))
2321 2322 2323 2324
        else:
            self.generate_guarded_basetype_call(
                base_type, None, "tp_setattro", "o, n, v", code)
            code.putln(
2325
                "return PyObject_GenericSetAttr(o, n, v);")
2326
        code.putln(
2327
            "}")
2328
        code.putln(
2329
            "else {")
2330 2331
        if del_entry:
            code.putln(
2332 2333
                "return %s(o, n);" % (
                    del_entry.func_cname))
2334 2335 2336 2337
        else:
            self.generate_guarded_basetype_call(
                base_type, None, "tp_setattro", "o, n, v", code)
            code.putln(
2338
                "return PyObject_GenericSetAttr(o, n, 0);")
2339
        code.putln(
2340
            "}")
2341 2342
        code.putln(
            "}")
2343

2344 2345 2346 2347 2348 2349 2350 2351
    def generate_descr_get_function(self, scope, code):
        # The __get__ function of a descriptor object can be
        # called with NULL for the second or third arguments
        # under some circumstances, so we replace them with
        # None in that case.
        user_get_entry = scope.lookup_here("__get__")
        code.putln("")
        code.putln(
2352 2353
            "static PyObject *%s(PyObject *o, PyObject *i, PyObject *c) {" % (
                scope.mangle_internal("tp_descr_get")))
2354 2355 2356 2357 2358 2359 2360 2361 2362
        code.putln(
            "PyObject *r = 0;")
        code.putln(
            "if (!i) i = Py_None;")
        code.putln(
            "if (!c) c = Py_None;")
        #code.put_incref("i", py_object_type)
        #code.put_incref("c", py_object_type)
        code.putln(
2363 2364
            "r = %s(o, i, c);" % (
                user_get_entry.func_cname))
2365 2366 2367 2368 2369 2370
        #code.put_decref("i", py_object_type)
        #code.put_decref("c", py_object_type)
        code.putln(
            "return r;")
        code.putln(
            "}")
2371

2372 2373 2374 2375 2376 2377 2378 2379 2380
    def generate_descr_set_function(self, scope, code):
        # Setting and deleting are both done through the __set__
        # method of a descriptor, so we dispatch to user's __set__
        # or __delete__ or raise an exception.
        base_type = scope.parent_type.base_type
        user_set_entry = scope.lookup_here("__set__")
        user_del_entry = scope.lookup_here("__delete__")
        code.putln("")
        code.putln(
2381 2382
            "static int %s(PyObject *o, PyObject *i, PyObject *v) {" % (
                scope.mangle_internal("tp_descr_set")))
2383
        code.putln(
2384
            "if (v) {")
2385 2386
        if user_set_entry:
            code.putln(
2387 2388
                "return %s(o, i, v);" % (
                    user_set_entry.func_cname))
2389 2390 2391 2392
        else:
            self.generate_guarded_basetype_call(
                base_type, None, "tp_descr_set", "o, i, v", code)
            code.putln(
2393
                'PyErr_SetString(PyExc_NotImplementedError, "__set__");')
2394
            code.putln(
2395
                "return -1;")
2396
        code.putln(
2397
            "}")
2398
        code.putln(
2399
            "else {")
2400 2401
        if user_del_entry:
            code.putln(
2402 2403
                "return %s(o, i);" % (
                    user_del_entry.func_cname))
2404 2405 2406 2407
        else:
            self.generate_guarded_basetype_call(
                base_type, None, "tp_descr_set", "o, i, v", code)
            code.putln(
2408
                'PyErr_SetString(PyExc_NotImplementedError, "__delete__");')
2409
            code.putln(
2410
                "return -1;")
2411
        code.putln(
2412
            "}")
2413 2414
        code.putln(
            "}")
2415

2416 2417 2418 2419 2420 2421 2422
    def generate_property_accessors(self, cclass_scope, code):
        for entry in cclass_scope.property_entries:
            property_scope = entry.scope
            if property_scope.defines_any(["__get__"]):
                self.generate_property_get_function(entry, code)
            if property_scope.defines_any(["__set__", "__del__"]):
                self.generate_property_set_function(entry, code)
2423

2424 2425 2426 2427 2428 2429 2430
    def generate_property_get_function(self, property_entry, code):
        property_scope = property_entry.scope
        property_entry.getter_cname = property_scope.parent_scope.mangle(
            Naming.prop_get_prefix, property_entry.name)
        get_entry = property_scope.lookup_here("__get__")
        code.putln("")
        code.putln(
2431 2432
            "static PyObject *%s(PyObject *o, CYTHON_UNUSED void *x) {" % (
                property_entry.getter_cname))
2433
        code.putln(
2434 2435
            "return %s(o);" % (
                get_entry.func_cname))
2436 2437
        code.putln(
            "}")
2438

2439 2440 2441 2442 2443 2444 2445 2446
    def generate_property_set_function(self, property_entry, code):
        property_scope = property_entry.scope
        property_entry.setter_cname = property_scope.parent_scope.mangle(
            Naming.prop_set_prefix, property_entry.name)
        set_entry = property_scope.lookup_here("__set__")
        del_entry = property_scope.lookup_here("__del__")
        code.putln("")
        code.putln(
2447 2448
            "static int %s(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) {" % (
                property_entry.setter_cname))
2449
        code.putln(
2450
            "if (v) {")
2451 2452
        if set_entry:
            code.putln(
2453 2454
                "return %s(o, v);" % (
                    set_entry.func_cname))
2455 2456
        else:
            code.putln(
2457
                'PyErr_SetString(PyExc_NotImplementedError, "__set__");')
2458
            code.putln(
2459
                "return -1;")
2460
        code.putln(
2461
            "}")
2462
        code.putln(
2463
            "else {")
2464 2465
        if del_entry:
            code.putln(
2466 2467
                "return %s(o);" % (
                    del_entry.func_cname))
2468 2469
        else:
            code.putln(
2470
                'PyErr_SetString(PyExc_NotImplementedError, "__del__");')
2471
            code.putln(
2472
                "return -1;")
2473
        code.putln(
2474
            "}")
2475 2476 2477
        code.putln(
            "}")

2478 2479 2480 2481
    def generate_typeobj_spec(self, entry, code):
        ext_type = entry.type
        scope = ext_type.scope
        code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname)
2482 2483 2484 2485 2486
        for slot in TypeSlots.slot_table:
            slot.generate_spec(scope, code)
        code.putln("{0, 0},")
        code.putln("};")

2487 2488
        if ext_type.typedef_flag:
            objstruct = ext_type.objstruct_cname
2489
        else:
2490
            objstruct = "struct %s" % ext_type.objstruct_cname
2491
        classname = scope.class_name.as_c_string_literal()
2492
        code.putln("static PyType_Spec %s_spec = {" % ext_type.typeobj_cname)
2493 2494 2495 2496
        code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', '')))
        code.putln("sizeof(%s)," % objstruct)
        code.putln("0,")
        code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags").slot_code(scope))
2497
        code.putln("%s_slots," % ext_type.typeobj_cname)
2498 2499
        code.putln("};")

2500 2501 2502 2503 2504 2505 2506 2507 2508
    def generate_typeobj_definition(self, modname, entry, code):
        type = entry.type
        scope = type.scope
        for suite in TypeSlots.substructures:
            suite.generate_substructure(scope, code)
        code.putln("")
        if entry.visibility == 'public':
            header = "DL_EXPORT(PyTypeObject) %s = {"
        else:
2509
            header = "static PyTypeObject %s = {"
2510 2511 2512
        #code.putln(header % scope.parent_type.typeobj_cname)
        code.putln(header % type.typeobj_cname)
        code.putln(
2513
            "PyVarObject_HEAD_INIT(0, 0)")
2514
        classname = scope.class_name.as_c_string_literal()
2515
        code.putln(
2516 2517 2518
            '"%s."%s, /*tp_name*/' % (
                self.full_module_name,
                classname))
2519 2520 2521 2522 2523
        if type.typedef_flag:
            objstruct = type.objstruct_cname
        else:
            objstruct = "struct %s" % type.objstruct_cname
        code.putln(
2524
            "sizeof(%s), /*tp_basicsize*/" % objstruct)
2525 2526 2527 2528 2529 2530
        code.putln(
            "0, /*tp_itemsize*/")
        for slot in TypeSlots.slot_table:
            slot.generate(scope, code)
        code.putln(
            "};")
2531

2532
    def generate_method_table(self, env, code):
2533
        if env.is_c_class_scope and not env.pyfunc_entries:
2534
            return
2535
        binding = env.directives['binding']
2536

2537
        code.putln("")
2538 2539
        wrapper_code_writer = code.insertion_point()

2540
        code.putln(
2541 2542
            "static PyMethodDef %s[] = {" % (
                env.method_table_cname))
2543
        for entry in env.pyfunc_entries:
2544
            if not entry.fused_cfunction and not (binding and entry.is_overridable):
2545
                code.put_pymethoddef(entry, ",", wrapper_code_writer=wrapper_code_writer)
2546
        code.putln(
2547
            "{0, 0, 0, 0}")
2548 2549
        code.putln(
            "};")
2550

2551 2552 2553
        if wrapper_code_writer.getvalue():
            wrapper_code_writer.putln("")

2554
    def generate_dict_getter_function(self, scope, code):
2555
        dict_attr = scope.lookup_here("__dict__")
2556 2557 2558
        if not dict_attr or not dict_attr.is_variable:
            return
        func_name = scope.mangle_internal("__dict__getter")
2559 2560 2561 2562
        dict_name = dict_attr.cname
        code.putln("")
        code.putln("static PyObject *%s(PyObject *o, CYTHON_UNUSED void *x) {" % func_name)
        self.generate_self_cast(scope, code)
2563
        code.putln("if (unlikely(!p->%s)){" % dict_name)
2564 2565
        code.putln("p->%s = PyDict_New();" % dict_name)
        code.putln("}")
2566
        code.putln("Py_XINCREF(p->%s);" % dict_name)
2567 2568 2569
        code.putln("return p->%s;" % dict_name)
        code.putln("}")

2570
    def generate_getset_table(self, env, code):
2571
        if env.property_entries:
2572 2573 2574
            code.putln("")
            code.putln(
                "static struct PyGetSetDef %s[] = {" %
Stefan Behnel's avatar
Stefan Behnel committed
2575
                env.getset_table_cname)
2576
            for entry in env.property_entries:
2577 2578 2579 2580
                doc = entry.doc
                if doc:
                    if doc.is_unicode:
                        doc = doc.as_utf8_string()
2581
                    doc_code = "PyDoc_STR(%s)" % doc.as_c_string_literal()
2582
                else:
2583 2584
                    doc_code = "0"
                code.putln(
2585 2586
                    '{(char *)%s, %s, %s, (char *)%s, 0},' % (
                        entry.name.as_c_string_literal(),
2587 2588 2589
                        entry.getter_cname or "0",
                        entry.setter_cname or "0",
                        doc_code))
2590
            code.putln(
Stefan Behnel's avatar
Stefan Behnel committed
2591
                "{0, 0, 0, 0, 0}")
2592 2593
            code.putln(
                "};")
2594

2595 2596 2597
    def create_import_star_conversion_utility_code(self, env):
        # Create all conversion helpers that are needed for "import *" assignments.
        # Must be done before code generation to support CythonUtilityCode.
2598
        for name, entry in sorted(env.entries.items()):
2599 2600 2601 2602
            if entry.is_cglobal and entry.used:
                if not entry.type.is_pyobject:
                    entry.type.create_from_py_utility_code(env)

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2603
    def generate_import_star(self, env, code):
2604
        env.use_utility_code(UtilityCode.load_cached("CStringEquals", "StringTools.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2605
        code.putln()
Stefan Behnel's avatar
Stefan Behnel committed
2606
        code.enter_cfunc_scope()  # as we need labels
2607 2608
        code.putln("static int %s(PyObject *o, PyObject* py_name, char *name) {" % Naming.import_star_set)

2609
        code.putln("static const char* internal_type_names[] = {")
2610
        for name, entry in sorted(env.entries.items()):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2611 2612 2613 2614
            if entry.is_type:
                code.putln('"%s",' % name)
        code.putln("0")
        code.putln("};")
2615

2616
        code.putln("const char** type_name = internal_type_names;")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2617
        code.putln("while (*type_name) {")
2618
        code.putln("if (__Pyx_StrEq(name, *type_name)) {")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2619 2620 2621 2622 2623
        code.putln('PyErr_Format(PyExc_TypeError, "Cannot overwrite C type %s", name);')
        code.putln('goto bad;')
        code.putln("}")
        code.putln("type_name++;")
        code.putln("}")
2624

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2625
        old_error_label = code.new_error_label()
Stefan Behnel's avatar
Stefan Behnel committed
2626
        code.putln("if (0);")  # so the first one can be "else if"
2627
        msvc_count = 0
2628
        for name, entry in sorted(env.entries.items()):
2629
            if entry.is_cglobal and entry.used and not entry.type.is_const:
2630 2631 2632 2633 2634
                msvc_count += 1
                if msvc_count % 100 == 0:
                    code.putln("#ifdef _MSC_VER")
                    code.putln("if (0);  /* Workaround for MSVC C1061. */")
                    code.putln("#endif")
2635
                code.putln('else if (__Pyx_StrEq(name, "%s")) {' % name)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2636 2637 2638 2639 2640
                if entry.type.is_pyobject:
                    if entry.type.is_extension_type or entry.type.is_builtin_type:
                        code.putln("if (!(%s)) %s;" % (
                            entry.type.type_test_code("o"),
                            code.error_goto(entry.pos)))
2641 2642
                    code.putln("Py_INCREF(o);")
                    code.put_decref(entry.cname, entry.type, nanny=False)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2643
                    code.putln("%s = %s;" % (
2644
                        entry.cname,
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2645
                        PyrexTypes.typecast(entry.type, py_object_type, "o")))
2646 2647
                elif entry.type.create_from_py_utility_code(env):
                    # if available, utility code was already created in self.prepare_utility_code()
2648 2649
                    code.putln(entry.type.from_py_call_code(
                        'o', entry.cname, entry.pos, code))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2650
                else:
Stefan Behnel's avatar
Stefan Behnel committed
2651 2652
                    code.putln('PyErr_Format(PyExc_TypeError, "Cannot convert Python object %s to %s");' % (
                        name, entry.type))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2653 2654 2655 2656 2657 2658
                    code.putln(code.error_goto(entry.pos))
                code.putln("}")
        code.putln("else {")
        code.putln("if (PyObject_SetAttr(%s, py_name, o) < 0) goto bad;" % Naming.module_cname)
        code.putln("}")
        code.putln("return 0;")
2659 2660 2661
        if code.label_used(code.error_label):
            code.put_label(code.error_label)
            # This helps locate the offending name.
2662
            code.put_add_traceback(EncodedString(self.full_module_name))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2663 2664 2665 2666
        code.error_label = old_error_label
        code.putln("bad:")
        code.putln("return -1;")
        code.putln("}")
2667
        code.putln("")
2668
        code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1])
2669
        code.exit_cfunc_scope()  # done with labels
2670

2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681
    def generate_module_state_start(self, env, code):
        # TODO: Reactor LIMITED_API struct decl closer to the static decl
        code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
        code.putln('typedef struct {')
        code.putln('PyObject *%s;' % Naming.builtins_cname)
        code.putln('PyObject *%s;' % Naming.cython_runtime_cname)
        code.putln('PyObject *%s;' % Naming.empty_tuple)
        code.putln('PyObject *%s;' % Naming.empty_bytes)
        code.putln('PyObject *%s;' % Naming.empty_unicode)
        if Options.pre_import is not None:
            code.putln('PyObject *%s;' % Naming.preimport_cname)
2682
        code.putln('#ifdef __Pyx_CyFunction_USED')
2683
        code.putln('PyTypeObject *%s;' % Naming.cyfunction_type_cname)
2684 2685
        code.putln('#endif')
        code.putln('#ifdef __Pyx_FusedFunction_USED')
2686
        code.putln('PyTypeObject *%s;' % Naming.fusedfunction_type_cname)
2687
        code.putln('#endif')
2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718

    def generate_module_state_end(self, env, modules, globalstate):
        module_state = globalstate['module_state']
        module_state_defines = globalstate['module_state_defines']
        module_state_clear = globalstate['module_state_clear']
        module_state_traverse = globalstate['module_state_traverse']
        module_state.putln('} %s;' % Naming.modulestate_cname)
        module_state.putln('')
        module_state.putln('#ifdef __cplusplus')
        module_state.putln('namespace {')
        module_state.putln('extern struct PyModuleDef %s;' % Naming.pymoduledef_cname)
        module_state.putln('} /* anonymous namespace */')
        module_state.putln('#else')
        module_state.putln('static struct PyModuleDef %s;' % Naming.pymoduledef_cname)
        module_state.putln('#endif')
        module_state.putln('')
        module_state.putln('#define %s(o) ((%s *)__Pyx_PyModule_GetState(o))' % (
            Naming.modulestate_cname,
            Naming.modulestate_cname))
        module_state.putln('')
        module_state.putln('#define %s (%s(PyState_FindModule(&%s)))' % (
            Naming.modulestateglobal_cname,
            Naming.modulestate_cname,
            Naming.pymoduledef_cname))
        module_state.putln('')
        module_state.putln('#define %s (PyState_FindModule(&%s))' % (
            env.module_cname,
            Naming.pymoduledef_cname))
        module_state.putln("#endif")
        module_state_defines.putln("#endif")
        module_state_clear.putln("return 0;")
2719
        module_state_clear.putln("}")
2720 2721
        module_state_clear.putln("#endif")
        module_state_traverse.putln("return 0;")
2722
        module_state_traverse.putln("}")
2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751
        module_state_traverse.putln("#endif")

    def generate_module_state_defines(self, env, code):
        code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
        code.putln('#define %s %s->%s' % (
            Naming.builtins_cname,
            Naming.modulestateglobal_cname,
            Naming.builtins_cname))
        code.putln('#define %s %s->%s' % (
            Naming.cython_runtime_cname,
            Naming.modulestateglobal_cname,
            Naming.cython_runtime_cname))
        code.putln('#define %s %s->%s' % (
            Naming.empty_tuple,
            Naming.modulestateglobal_cname,
            Naming.empty_tuple))
        code.putln('#define %s %s->%s' % (
            Naming.empty_bytes,
            Naming.modulestateglobal_cname,
            Naming.empty_bytes))
        code.putln('#define %s %s->%s' % (
            Naming.empty_unicode,
            Naming.modulestateglobal_cname,
            Naming.empty_unicode))
        if Options.pre_import is not None:
            code.putln('#define %s %s->%s' % (
                Naming.preimport_cname,
                Naming.modulestateglobal_cname,
                Naming.preimport_cname))
2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763
        code.putln('#ifdef __Pyx_CyFunction_USED')
        code.putln('#define %s %s->%s' % (
            Naming.cyfunction_type_cname,
            Naming.modulestateglobal_cname,
            Naming.cyfunction_type_cname))
        code.putln('#endif')
        code.putln('#ifdef __Pyx_FusedFunction_USED')
        code.putln('#define %s %s->%s' %
            (Naming.fusedfunction_type_cname,
            Naming.modulestateglobal_cname,
            Naming.fusedfunction_type_cname))
        code.putln('#endif')
2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781

    def generate_module_state_clear(self, env, code):
        code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
        code.putln("static int %s_clear(PyObject *m) {" % Naming.module_cname)
        code.putln("%s *clear_module_state = %s(m);" % (
            Naming.modulestate_cname,
            Naming.modulestate_cname))
        code.putln("if (!clear_module_state) return 0;")
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.builtins_cname)
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.cython_runtime_cname)
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.empty_tuple)
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.empty_bytes)
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.empty_unicode)
2782 2783 2784 2785 2786 2787 2788 2789
        code.putln('#ifdef __Pyx_CyFunction_USED')
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.cyfunction_type_cname)
        code.putln('#endif')
        code.putln('#ifdef __Pyx_FusedFunction_USED')
        code.putln('Py_CLEAR(clear_module_state->%s);' %
            Naming.fusedfunction_type_cname)
        code.putln('#endif')
2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807

    def generate_module_state_traverse(self, env, code):
        code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
        code.putln("static int %s_traverse(PyObject *m, visitproc visit, void *arg) {" % Naming.module_cname)
        code.putln("%s *traverse_module_state = %s(m);" % (
            Naming.modulestate_cname,
            Naming.modulestate_cname))
        code.putln("if (!traverse_module_state) return 0;")
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.builtins_cname)
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.cython_runtime_cname)
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.empty_tuple)
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.empty_bytes)
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.empty_unicode)
2808 2809 2810 2811 2812 2813 2814 2815
        code.putln('#ifdef __Pyx_CyFunction_USED')
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.cyfunction_type_cname)
        code.putln('#endif')
        code.putln('#ifdef __Pyx_FusedFunction_USED')
        code.putln('Py_VISIT(traverse_module_state->%s);' %
            Naming.fusedfunction_type_cname)
        code.putln('#endif')
2816

2817
    def generate_module_init_func(self, imported_modules, env, code):
2818
        subfunction = self.mod_init_subfunction(self.pos, self.scope, code)
2819

2820 2821
        self.generate_pymoduledef_struct(env, code)

2822
        code.enter_cfunc_scope(self.scope)
2823
        code.putln("")
2824
        code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0])
da-woods's avatar
da-woods committed
2825 2826 2827 2828 2829 2830 2831 2832 2833
        if env.module_name.isascii():
            py2_mod_name = env.module_name
            fail_compilation_in_py2 = False
        else:
            fail_compilation_in_py2 = True
            # at this point py2_mod_name is largely a placeholder and the value doesn't matter
            py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf8")

        header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_mod_name
2834
        header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env)
2835
        header3 = EncodedString(header3)
2836
        code.putln("#if PY_MAJOR_VERSION < 3")
2837
        # Optimise for small code size as the module init function is only executed once.
2838
        code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header2)
da-woods's avatar
da-woods committed
2839 2840
        if fail_compilation_in_py2:
            code.putln('#error "Unicode module names are not supported in Python 2";')
2841 2842
        if self.scope.is_package:
            code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(WIN32) || defined(MS_WINDOWS))")
da-woods's avatar
da-woods committed
2843
            code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_mod_name)
2844
            code.putln("#endif")
2845 2846
        code.putln(header2)
        code.putln("#else")
2847
        code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header3)
2848 2849
        if self.scope.is_package:
            code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(WIN32) || defined(MS_WINDOWS))")
2850
            code.putln("__Pyx_PyMODINIT_FUNC PyInit___init__(void) { return %s(); }" % (
2851 2852
                self.mod_init_func_cname('PyInit', env)))
            code.putln("#endif")
2853
        code.putln(header3)
2854 2855 2856

        # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.)
        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2857
        code.putln("{")
2858 2859 2860
        code.putln("return PyModuleDef_Init(&%s);" % Naming.pymoduledef_cname)
        code.putln("}")

2861 2862
        mod_create_func = UtilityCode.load_as_string("ModuleCreationPEP489", "ModuleSetupCode.c")[1]
        code.put(mod_create_func)
2863 2864 2865

        code.putln("")
        # main module init code lives in Py_mod_exec function, not in PyInit function
2866
        code.putln("static CYTHON_SMALL_CODE int %s(PyObject *%s)" % (
2867
            self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env),
2868 2869 2870 2871 2872 2873 2874 2875
            Naming.pymodinit_module_arg))
        code.putln("#endif")  # PEP489

        code.putln("#endif")  # Py3

        # start of module init/exec function (pre/post PEP 489)
        code.putln("{")

2876
        tempdecl_code = code.insertion_point()
Robert Bradshaw's avatar
Robert Bradshaw committed
2877

2878 2879 2880
        profile = code.globalstate.directives['profile']
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
2881 2882
            if linetrace:
                code.use_fast_gil_utility_code()
2883 2884
            code.globalstate.use_utility_code(UtilityCode.load_cached("Profile", "Profile.c"))

2885
        code.put_declare_refcount_context()
2886
        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2887 2888 2889
        # Most extension modules simply can't deal with it, and Cython isn't ready either.
        # See issues listed here: https://docs.python.org/3/c-api/init.html#sub-interpreter-support
        code.putln("if (%s) {" % Naming.module_cname)
2890
        # Hack: enforce single initialisation.
2891
        code.putln("if (%s == %s) return 0;" % (
2892 2893 2894
            Naming.module_cname,
            Naming.pymodinit_module_arg,
        ))
2895 2896
        code.putln('PyErr_SetString(PyExc_RuntimeError,'
                   ' "Module \'%s\' has already been imported. Re-initialisation is not supported.");' %
da-woods's avatar
da-woods committed
2897
                   env.module_name.as_c_string_literal()[1:-1])
2898
        code.putln("return -1;")
2899
        code.putln("}")
2900 2901 2902 2903 2904 2905
        code.putln("#elif PY_MAJOR_VERSION >= 3")
        # Hack: enforce single initialisation also on reimports under different names on Python 3 (with PEP 3121/489).
        code.putln("if (%s) return __Pyx_NewRef(%s);" % (
            Naming.module_cname,
            Naming.module_cname,
        ))
2906 2907
        code.putln("#endif")

2908 2909 2910
        code.putln("/*--- Module creation code ---*/")
        self.generate_module_creation_code(env, code)

2911
        if profile or linetrace:
2912 2913
            tempdecl_code.put_trace_declarations()
            code.put_trace_frame_init()
2914

2915 2916
        refnanny_import_code = UtilityCode.load_as_string("ImportRefnannyAPI", "ModuleSetupCode.c")[1]
        code.putln(refnanny_import_code.rstrip())
2917
        code.put_setup_refcount_context(header3)
2918

2919
        env.use_utility_code(UtilityCode.load("CheckBinaryVersion", "ModuleSetupCode.c"))
2920
        code.put_error_if_neg(self.pos, "__Pyx_check_binary_version()")
2921

2922
        code.putln("#ifdef __Pxy_PyFrame_Initialize_Offsets")
2923
        code.putln("__Pxy_PyFrame_Initialize_Offsets();")
2924
        code.putln("#endif")
Stefan Behnel's avatar
Stefan Behnel committed
2925 2926 2927 2928
        code.putln("%s = PyTuple_New(0); %s" % (
            Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)))
        code.putln("%s = PyBytes_FromStringAndSize(\"\", 0); %s" % (
            Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos)))
2929 2930
        code.putln("%s = PyUnicode_FromStringAndSize(\"\", 0); %s" % (
            Naming.empty_unicode, code.error_goto_if_null(Naming.empty_unicode, self.pos)))
2931

2932
        for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'AsyncGen', 'StopAsyncIteration'):
2933 2934 2935
            code.putln("#ifdef __Pyx_%s_USED" % ext_type)
            code.put_error_if_neg(self.pos, "__pyx_%s_init()" % ext_type)
            code.putln("#endif")
2936

2937
        code.putln("/*--- Library function declarations ---*/")
2938
        if env.directives['np_pythran']:
2939
            code.put_error_if_neg(self.pos, "_import_array()")
2940

2941
        code.putln("/*--- Threads initialization code ---*/")
2942 2943
        code.putln("#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x030700F0 "
                   "&& defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS")
2944 2945 2946
        code.putln("PyEval_InitThreads();")
        code.putln("#endif")

2947
        code.putln("/*--- Initialize various global constants etc. ---*/")
2948
        code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()")
2949

2950 2951
        code.putln("#if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || "
                   "__PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT)")
2952
        code.put_error_if_neg(self.pos, "__Pyx_init_sys_getdefaultencoding_params()")
2953 2954
        code.putln("#endif")

da-woods's avatar
da-woods committed
2955
        code.putln("if (%s) {" % self.is_main_module_flag_cname())
2956
        code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % (
2957
            env.module_cname,
2958 2959
            code.intern_identifier(EncodedString("__name__")),
            code.intern_identifier(EncodedString("__main__"))))
2960
        code.putln("}")
2961

2962 2963
        # set up __file__ and __path__, then add the module to sys.modules
        self.generate_module_import_setup(env, code)
2964

Robert Bradshaw's avatar
Robert Bradshaw committed
2965 2966
        if Options.cache_builtins:
            code.putln("/*--- Builtin init code ---*/")
2967
            code.put_error_if_neg(self.pos, "__Pyx_InitCachedBuiltins()")
2968 2969

        code.putln("/*--- Constants init code ---*/")
2970
        code.put_error_if_neg(self.pos, "__Pyx_InitCachedConstants()")
2971

2972 2973 2974 2975
        code.putln("/*--- Global type/function init code ---*/")

        with subfunction("Global init code") as inner_code:
            self.generate_global_init_code(env, inner_code)
Gary Furnish's avatar
Gary Furnish committed
2976

2977 2978
        with subfunction("Variable export code") as inner_code:
            self.generate_c_variable_export_code(env, inner_code)
2979

2980 2981
        with subfunction("Function export code") as inner_code:
            self.generate_c_function_export_code(env, inner_code)
2982

2983 2984
        with subfunction("Type init code") as inner_code:
            self.generate_type_init_code(env, inner_code)
2985

2986 2987 2988
        with subfunction("Type import code") as inner_code:
            for module in imported_modules:
                self.generate_type_import_code_for_module(module, env, inner_code)
2989

2990 2991 2992
        with subfunction("Variable import code") as inner_code:
            for module in imported_modules:
                self.generate_c_variable_import_code_for_module(module, env, inner_code)
2993

2994 2995 2996 2997
        with subfunction("Function import code") as inner_code:
            for module in imported_modules:
                self.specialize_fused_types(module)
                self.generate_c_function_import_code_for_module(module, env, inner_code)
Gary Furnish's avatar
Gary Furnish committed
2998

2999
        code.putln("/*--- Execution code ---*/")
Robert Bradshaw's avatar
Robert Bradshaw committed
3000
        code.mark_pos(None)
3001

3002
        code.putln("#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED)")
3003 3004 3005
        code.put_error_if_neg(self.pos, "__Pyx_patch_abc()")
        code.putln("#endif")

3006 3007 3008 3009
        if profile or linetrace:
            code.put_trace_call(header3, self.pos, nogil=not code.funcstate.gil_owned)
            code.funcstate.can_trace = True

3010
        self.body.generate_execution_code(code)
3011

3012 3013 3014 3015
        if profile or linetrace:
            code.funcstate.can_trace = False
            code.put_trace_return("Py_None", nogil=not code.funcstate.gil_owned)

3016 3017 3018 3019 3020
        code.putln()
        code.putln("/*--- Wrapped vars code ---*/")
        self.generate_wrapped_entries_code(env, code)
        code.putln()

3021
        if Options.generate_cleanup_code:
3022 3023
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RegisterModuleCleanup", "ModuleSetupCode.c"))
3024
            code.putln("if (__Pyx_RegisterCleanup()) %s;" % code.error_goto(self.pos))
3025

3026
        code.put_goto(code.return_label)
3027
        code.put_label(code.error_label)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3028 3029
        for cname, type in code.funcstate.all_managed_temps():
            code.put_xdecref(cname, type)
3030
        code.putln('if (%s) {' % env.module_cname)
3031
        code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
3032
        code.putln('if (%s) {' % env.module_dict_cname)
3033
        code.putln("#endif")
3034
        code.put_add_traceback(EncodedString("init %s" % env.qualified_name))
3035
        code.globalstate.use_utility_code(Nodes.traceback_utility_code)
3036 3037 3038 3039 3040
        # Module reference and module dict are in global variables which might still be needed
        # for cleanup, atexit code, etc., so leaking is better than crashing.
        # At least clearing the module dict here might be a good idea, but could still break
        # user code in atexit or other global registries.
        ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False)
3041
        code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
3042
        code.putln('}')
3043
        code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True)
3044
        code.putln("#endif")
3045
        code.putln('} else if (!PyErr_Occurred()) {')
da-woods's avatar
da-woods committed
3046 3047
        code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' %
                   env.qualified_name.as_c_string_literal()[1:-1])
3048
        code.putln('}')
3049
        code.put_label(code.return_label)
3050 3051 3052

        code.put_finish_refcount_context()

3053 3054 3055
        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
        code.putln("return (%s != NULL) ? 0 : -1;" % env.module_cname)
        code.putln("#elif PY_MAJOR_VERSION >= 3")
3056
        code.putln("return %s;" % env.module_cname)
3057 3058
        code.putln("#else")
        code.putln("return;")
3059
        code.putln("#endif")
3060
        code.putln('}')
3061

3062
        tempdecl_code.put_temp_declarations(code.funcstate)
3063

3064
        code.exit_cfunc_scope()
3065

3066
    def mod_init_subfunction(self, pos, scope, orig_code):
Stefan Behnel's avatar
Stefan Behnel committed
3067 3068 3069 3070 3071 3072 3073 3074
        """
        Return a context manager that allows deviating the module init code generation
        into a separate function and instead inserts a call to it.

        Can be reused sequentially to create multiple functions.
        The functions get inserted at the point where the context manager was created.
        The call gets inserted where the context manager is used (on entry).
        """
3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092
        prototypes = orig_code.insertion_point()
        prototypes.putln("")
        function_code = orig_code.insertion_point()
        function_code.putln("")

        class ModInitSubfunction(object):
            def __init__(self, code_type):
                cname = '_'.join(code_type.lower().split())
                assert re.match("^[a-z0-9_]+$", cname)
                self.cfunc_name = "__Pyx_modinit_%s" % cname
                self.description = code_type
                self.tempdecl_code = None
                self.call_code = None

            def __enter__(self):
                self.call_code = orig_code.insertion_point()
                code = function_code
                code.enter_cfunc_scope(scope)
3093
                prototypes.putln("static CYTHON_SMALL_CODE int %s(void); /*proto*/" % self.cfunc_name)
3094
                code.putln("static int %s(void) {" % self.cfunc_name)
3095 3096
                code.put_declare_refcount_context()
                self.tempdecl_code = code.insertion_point()
3097
                code.put_setup_refcount_context(EncodedString(self.cfunc_name))
3098 3099 3100 3101 3102 3103
                # Leave a grepable marker that makes it easy to find the generator source.
                code.putln("/*--- %s ---*/" % self.description)
                return code

            def __exit__(self, *args):
                code = function_code
3104
                code.put_finish_refcount_context()
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114
                code.putln("return 0;")

                self.tempdecl_code.put_temp_declarations(code.funcstate)
                self.tempdecl_code = None

                needs_error_handling = code.label_used(code.error_label)
                if needs_error_handling:
                    code.put_label(code.error_label)
                    for cname, type in code.funcstate.all_managed_temps():
                        code.put_xdecref(cname, type)
3115
                    code.put_finish_refcount_context()
3116 3117 3118 3119 3120 3121
                    code.putln("return -1;")
                code.putln("}")
                code.exit_cfunc_scope()
                code.putln("")

                if needs_error_handling:
3122 3123
                    self.call_code.putln(
                        self.call_code.error_goto_if_neg("%s()" % self.cfunc_name, pos))
3124 3125 3126 3127 3128 3129
                else:
                    self.call_code.putln("(void)%s();" % self.cfunc_name)
                self.call_code = None

        return ModInitSubfunction

3130
    def generate_module_import_setup(self, env, code):
3131 3132 3133
        module_path = env.directives['set_initial_path']
        if module_path == 'SOURCEFILE':
            module_path = self.pos[0].filename
3134 3135

        if module_path:
3136
            code.putln('if (!CYTHON_PEP489_MULTI_PHASE_INIT) {')
3137
            code.putln('if (PyObject_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
3138
                env.module_cname,
3139 3140
                code.globalstate.get_py_string_const(
                    EncodedString(decode_filename(module_path))).cname,
3141
                code.error_goto(self.pos)))
3142
            code.putln("}")
3143 3144 3145

            if env.is_package:
                # set __path__ to mark the module as package
3146
                code.putln('if (!CYTHON_PEP489_MULTI_PHASE_INIT) {')
3147 3148 3149 3150 3151 3152 3153
                temp = code.funcstate.allocate_temp(py_object_type, True)
                code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
                    temp,
                    code.globalstate.get_py_string_const(
                        EncodedString(decode_filename(
                            os.path.dirname(module_path)))).cname,
                    code.error_goto_if_null(temp, self.pos)))
3154
                code.put_gotref(temp, py_object_type)
3155
                code.putln(
3156
                    'if (PyObject_SetAttrString(%s, "__path__", %s) < 0) %s;' % (
3157 3158 3159
                        env.module_cname, temp, code.error_goto(self.pos)))
                code.put_decref_clear(temp, py_object_type)
                code.funcstate.release_temp(temp)
3160
                code.putln("}")
3161 3162 3163 3164

        elif env.is_package:
            # packages require __path__, so all we can do is try to figure
            # out the module path at runtime by rerunning the import lookup
3165
            code.putln("if (!CYTHON_PEP489_MULTI_PHASE_INIT) {")
3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
            package_name, _ = self.full_module_name.rsplit('.', 1)
            if '.' in package_name:
                parent_name = '"%s"' % (package_name.rsplit('.', 1)[0],)
            else:
                parent_name = 'NULL'
            code.globalstate.use_utility_code(UtilityCode.load(
                "SetPackagePathFromImportLib", "ImportExport.c"))
            code.putln(code.error_goto_if_neg(
                '__Pyx_SetPackagePathFromImportLib(%s, %s)' % (
                    parent_name,
                    code.globalstate.get_py_string_const(
                        EncodedString(env.module_name)).cname),
                self.pos))
3179
            code.putln("}")
3180 3181 3182 3183

        # CPython may not have put us into sys.modules yet, but relative imports and reimports require it
        fq_module_name = self.full_module_name
        if fq_module_name.endswith('.__init__'):
da-woods's avatar
da-woods committed
3184 3185
            fq_module_name = EncodedString(fq_module_name[:-len('.__init__')])
        fq_module_name_cstring = fq_module_name.as_c_string_literal()
3186 3187 3188 3189
        code.putln("#if PY_MAJOR_VERSION >= 3")
        code.putln("{")
        code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" %
                   code.error_goto_if_null("modules", self.pos))
da-woods's avatar
da-woods committed
3190 3191 3192
        code.putln('if (!PyDict_GetItemString(modules, %s)) {' % fq_module_name_cstring)
        code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, %s, %s)' % (
            fq_module_name_cstring, env.module_cname), self.pos))
3193 3194 3195
        code.putln("}")
        code.putln("}")
        code.putln("#endif")
3196

3197 3198 3199
    def generate_module_cleanup_func(self, env, code):
        if not Options.generate_cleanup_code:
            return
3200

3201
        code.putln('static void %s(CYTHON_UNUSED PyObject *self) {' %
3202
                   Naming.cleanup_cname)
3203 3204
        if Options.generate_cleanup_code >= 2:
            code.putln("/*--- Global cleanup code ---*/")
3205 3206 3207
            rev_entries = list(env.var_entries)
            rev_entries.reverse()
            for entry in rev_entries:
3208
                if entry.visibility != 'extern':
3209
                    if entry.type.is_pyobject and entry.used:
3210 3211 3212 3213
                        code.put_xdecref_clear(
                            entry.cname, entry.type,
                            clear_before_decref=True,
                            nanny=False)
3214
        code.putln("__Pyx_CleanupGlobals();")
3215 3216
        if Options.generate_cleanup_code >= 3:
            code.putln("/*--- Type import cleanup code ---*/")
3217
            for ext_type in sorted(env.types_imported, key=operator.attrgetter('typeptr_cname')):
3218
                code.put_xdecref_clear(
3219
                    ext_type.typeptr_cname, ext_type,
3220 3221
                    clear_before_decref=True,
                    nanny=False)
3222 3223
        if Options.cache_builtins:
            code.putln("/*--- Builtin cleanup code ---*/")
3224
            for entry in env.cached_builtins:
3225 3226 3227 3228
                code.put_xdecref_clear(
                    entry.cname, PyrexTypes.py_object_type,
                    clear_before_decref=True,
                    nanny=False)
3229
        code.putln("/*--- Intern cleanup code ---*/")
3230 3231
        code.put_decref_clear(Naming.empty_tuple,
                              PyrexTypes.py_object_type,
3232
                              clear_before_decref=True,
3233
                              nanny=False)
3234 3235
        for entry in env.c_class_entries:
            cclass_type = entry.type
3236
            if cclass_type.is_external or cclass_type.base_type:
3237 3238 3239 3240 3241 3242 3243 3244 3245 3246
                continue
            if cclass_type.scope.directives.get('freelist', 0):
                scope = cclass_type.scope
                freelist_name = scope.mangle_internal(Naming.freelist_name)
                freecount_name = scope.mangle_internal(Naming.freecount_name)
                code.putln("while (%s > 0) {" % freecount_name)
                code.putln("PyObject* o = (PyObject*)%s[--%s];" % (
                    freelist_name, freecount_name))
                code.putln("(*Py_TYPE(o)->tp_free)(o);")
                code.putln("}")
3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259
#        for entry in env.pynum_entries:
#            code.put_decref_clear(entry.cname,
#                                  PyrexTypes.py_object_type,
#                                  nanny=False)
#        for entry in env.all_pystring_entries:
#            if entry.is_interned:
#                code.put_decref_clear(entry.pystring_cname,
#                                      PyrexTypes.py_object_type,
#                                      nanny=False)
#        for entry in env.default_entries:
#            if entry.type.is_pyobject and entry.used:
#                code.putln("Py_DECREF(%s); %s = 0;" % (
#                    code.entry_as_pyobject(entry), entry.cname))
3260 3261 3262
        if Options.pre_import is not None:
            code.put_decref_clear(Naming.preimport_cname, py_object_type,
                                  nanny=False, clear_before_decref=True)
3263
        for cname in [Naming.cython_runtime_cname, Naming.builtins_cname]:
3264
            code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True)
3265 3266 3267
        code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
        code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False, clear_before_decref=True)
        code.putln("#endif")
3268

3269
    def generate_main_method(self, env, code):
da-woods's avatar
da-woods committed
3270
        module_is_main = self.is_main_module_flag_cname()
3271 3272 3273 3274
        if Options.embed == "main":
            wmain = "wmain"
        else:
            wmain = Options.embed
3275
        main_method = UtilityCode.load_cached("MainFunction", "Embed.c")
3276 3277
        code.globalstate.use_utility_code(
            main_method.specialize(
Stefan Behnel's avatar
Stefan Behnel committed
3278 3279 3280 3281
                module_name=env.module_name,
                module_is_main=module_is_main,
                main_method=Options.embed,
                wmain_method=wmain))
3282

da-woods's avatar
da-woods committed
3283 3284 3285 3286 3287 3288 3289 3290
    def punycode_module_name(self, prefix, name):
        # adapted from PEP483
        try:
            name = '_' + name.encode('ascii').decode('ascii')
        except UnicodeEncodeError:
            name = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii')
        return "%s%s" % (prefix, name)

3291
    def mod_init_func_cname(self, prefix, env):
da-woods's avatar
da-woods committed
3292 3293
        # from PEP483
        return self.punycode_module_name(prefix, env.module_name)
3294

3295 3296
    def generate_pymoduledef_struct(self, env, code):
        if env.doc:
3297
            doc = "%s" % code.get_string_const(env.doc)
3298 3299
        else:
            doc = "0"
3300
        if Options.generate_cleanup_code:
Stefan Behnel's avatar
Stefan Behnel committed
3301
            cleanup_func = "(freefunc)%s" % Naming.cleanup_cname
3302 3303
        else:
            cleanup_func = 'NULL'
3304 3305 3306

        code.putln("")
        code.putln("#if PY_MAJOR_VERSION >= 3")
3307
        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
3308
        exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env)
3309 3310
        code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" %
                   Naming.pymodule_create_func_cname)
3311
        code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname)
3312 3313

        code.putln("static PyModuleDef_Slot %s[] = {" % Naming.pymoduledef_slots_cname)
Stefan Behnel's avatar
Stefan Behnel committed
3314 3315
        code.putln("{Py_mod_create, (void*)%s}," % Naming.pymodule_create_func_cname)
        code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname)
3316 3317
        code.putln("{0, NULL}")
        code.putln("};")
da-woods's avatar
da-woods committed
3318 3319 3320 3321
        if not env.module_name.isascii():
            code.putln("#else /* CYTHON_PEP489_MULTI_PHASE_INIT */")
            code.putln('#error "Unicode module names are only supported with multi-phase init'
                       ' as per PEP489"')
3322 3323 3324
        code.putln("#endif")

        code.putln("")
3325 3326
        code.putln('#ifdef __cplusplus')
        code.putln('namespace {')
3327
        code.putln("struct PyModuleDef %s =" % Naming.pymoduledef_cname)
3328
        code.putln('#else')
3329
        code.putln("static struct PyModuleDef %s =" % Naming.pymoduledef_cname)
3330
        code.putln('#endif')
3331
        code.putln('{')
3332
        code.putln("  PyModuleDef_HEAD_INIT,")
da-woods's avatar
da-woods committed
3333
        code.putln('  %s,' % env.module_name.as_c_string_literal())
3334
        code.putln("  %s, /* m_doc */" % doc)
3335 3336
        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
        code.putln("  0, /* m_size */")
3337 3338
        code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API")
        code.putln("  sizeof(%s), /* m_size */" % Naming.modulestate_cname)
3339
        code.putln("#else")
3340
        code.putln("  -1, /* m_size */")
3341
        code.putln("#endif")
3342
        code.putln("  %s /* m_methods */," % env.method_table_cname)
3343 3344 3345
        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
        code.putln("  %s, /* m_slots */" % Naming.pymoduledef_slots_cname)
        code.putln("#else")
3346
        code.putln("  NULL, /* m_reload */")
3347
        code.putln("#endif")
3348 3349 3350 3351 3352
        code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
        code.putln("  %s_traverse, /* m_traverse */" % Naming.module_cname)
        code.putln("  %s_clear, /* m_clear */" % Naming.module_cname)
        code.putln("  %s /* m_free */" % cleanup_func)
        code.putln("#else")
3353 3354
        code.putln("  NULL, /* m_traverse */")
        code.putln("  NULL, /* m_clear */")
Stefan Behnel's avatar
Stefan Behnel committed
3355
        code.putln("  %s /* m_free */" % cleanup_func)
3356
        code.putln("#endif")
3357
        code.putln("};")
3358 3359 3360
        code.putln('#ifdef __cplusplus')
        code.putln('} /* anonymous namespace */')
        code.putln('#endif')
3361 3362
        code.putln("#endif")

3363 3364 3365 3366
    def generate_module_creation_code(self, env, code):
        # Generate code to create the module object and
        # install the builtins.
        if env.doc:
3367
            doc = "%s" % code.get_string_const(env.doc)
3368 3369
        else:
            doc = "0"
3370 3371 3372 3373 3374 3375 3376

        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
        code.putln("%s = %s;" % (
            env.module_cname,
            Naming.pymodinit_module_arg))
        code.put_incref(env.module_cname, py_object_type, nanny=False)
        code.putln("#else")
3377
        code.putln("#if PY_MAJOR_VERSION < 3")
3378
        code.putln(
da-woods's avatar
da-woods committed
3379
            '%s = Py_InitModule4(%s, %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % (
3380
                env.module_cname,
da-woods's avatar
da-woods committed
3381
                env.module_name.as_c_string_literal(),
3382
                env.method_table_cname,
3383 3384
                doc,
                env.module_cname))
3385
        code.putln(code.error_goto_if_null(env.module_cname, self.pos))
3386 3387 3388 3389 3390 3391 3392
        code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API")
        module_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        code.putln(
            "%s = PyModule_Create(&%s); %s" % (
                module_temp,
                Naming.pymoduledef_cname,
                code.error_goto_if_null(module_temp, self.pos)))
3393
        code.put_gotref(module_temp, py_object_type)
3394 3395 3396 3397 3398
        code.putln(code.error_goto_if_neg("PyState_AddModule(%s, &%s)" % (
            module_temp, Naming.pymoduledef_cname), self.pos))
        code.put_decref_clear(module_temp, type=py_object_type)
        code.funcstate.release_temp(module_temp)
        code.putln('#else')
3399 3400 3401 3402
        code.putln(
            "%s = PyModule_Create(&%s);" % (
                env.module_cname,
                Naming.pymoduledef_cname))
3403
        code.putln(code.error_goto_if_null(env.module_cname, self.pos))
3404
        code.putln("#endif")
3405 3406
        code.putln("#endif")  # CYTHON_PEP489_MULTI_PHASE_INIT

3407
        code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
3408 3409 3410 3411 3412
        code.putln(
            "%s = PyModule_GetDict(%s); %s" % (
                env.module_dict_cname, env.module_cname,
                code.error_goto_if_null(env.module_dict_cname, self.pos)))
        code.put_incref(env.module_dict_cname, py_object_type, nanny=False)
3413
        code.putln("#endif")
3414

3415
        code.putln(
3416
            '%s = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); %s' % (
3417
                Naming.builtins_cname,
3418
                code.error_goto_if_null(Naming.builtins_cname, self.pos)))
3419
        code.put_incref(Naming.builtins_cname, py_object_type, nanny=False)
3420 3421 3422 3423
        code.putln(
            '%s = PyImport_AddModule((char *) "cython_runtime"); %s' % (
                Naming.cython_runtime_cname,
                code.error_goto_if_null(Naming.cython_runtime_cname, self.pos)))
3424
        code.put_incref(Naming.cython_runtime_cname, py_object_type, nanny=False)
3425
        code.putln(
3426
            'if (PyObject_SetAttrString(%s, "__builtins__", %s) < 0) %s;' % (
3427 3428 3429
                env.module_cname,
                Naming.builtins_cname,
                code.error_goto(self.pos)))
3430 3431
        if Options.pre_import is not None:
            code.putln(
3432
                '%s = PyImport_AddModule("%s"); %s' % (
3433
                    Naming.preimport_cname,
3434 3435
                    Options.pre_import,
                    code.error_goto_if_null(Naming.preimport_cname, self.pos)))
3436
            code.put_incref(Naming.preimport_cname, py_object_type, nanny=False)
3437

3438 3439 3440 3441
    def generate_global_init_code(self, env, code):
        # Generate code to initialise global PyObject *
        # variables to None.
        for entry in env.var_entries:
3442
            if entry.visibility != 'extern':
3443 3444
                if entry.used:
                    entry.type.global_init_code(entry, code)
3445

3446
    def generate_wrapped_entries_code(self, env, code):
3447
        for name, entry in sorted(env.entries.items()):
3448 3449 3450
            if (entry.create_wrapper
                    and not entry.is_type
                    and entry.scope is env):
3451 3452 3453 3454 3455 3456 3457 3458
                if not entry.type.create_to_py_utility_code(env):
                    error(entry.pos, "Cannot convert '%s' to Python object" % entry.type)
                code.putln("{")
                code.putln("PyObject* wrapped = %s(%s);"  % (
                    entry.type.to_py_function,
                    entry.cname))
                code.putln(code.error_goto_if_null("wrapped", entry.pos))
                code.putln(
3459
                    'if (PyObject_SetAttrString(%s, "%s", wrapped) < 0) %s;' % (
3460 3461 3462 3463 3464
                        env.module_cname,
                        name,
                        code.error_goto(entry.pos)))
                code.putln("}")

3465 3466
    def generate_c_variable_export_code(self, env, code):
        # Generate code to create PyCFunction wrappers for exported C functions.
3467
        entries = []
3468
        for entry in env.var_entries:
Robert Bradshaw's avatar
Robert Bradshaw committed
3469
            if (entry.api
3470 3471
                    or entry.defined_in_pxd
                    or (Options.cimport_from_pyx and not entry.visibility == 'extern')):
3472 3473
                entries.append(entry)
        if entries:
3474
            env.use_utility_code(UtilityCode.load_cached("VoidPtrExport", "ImportExport.c"))
3475
            for entry in entries:
3476
                signature = entry.type.empty_declaration_code()
3477 3478 3479
                name = code.intern_identifier(entry.name)
                code.putln('if (__Pyx_ExportVoidPtr(%s, (void *)&%s, "%s") < 0) %s' % (
                    name, entry.cname, signature,
3480 3481
                    code.error_goto(self.pos)))

3482 3483
    def generate_c_function_export_code(self, env, code):
        # Generate code to create PyCFunction wrappers for exported C functions.
3484
        entries = []
3485
        for entry in env.cfunc_entries:
Robert Bradshaw's avatar
Robert Bradshaw committed
3486
            if (entry.api
3487 3488
                    or entry.defined_in_pxd
                    or (Options.cimport_from_pyx and not entry.visibility == 'extern')):
3489 3490
                entries.append(entry)
        if entries:
3491 3492
            env.use_utility_code(
                UtilityCode.load_cached("FunctionExport", "ImportExport.c"))
Stefan Behnel's avatar
Stefan Behnel committed
3493 3494
            # Note: while this looks like it could be more cheaply stored and read from a struct array,
            # investigation shows that the resulting binary is smaller with repeated functions calls.
3495
            for entry in entries:
Mark Florisson's avatar
Mark Florisson committed
3496
                signature = entry.type.signature_string()
3497 3498
                code.putln('if (__Pyx_ExportFunction(%s, (void (*)(void))%s, "%s") < 0) %s' % (
                    entry.name.as_c_string_literal(),
Mark Florisson's avatar
Mark Florisson committed
3499 3500 3501
                    entry.cname,
                    signature,
                    code.error_goto(self.pos)))
3502

3503
    def generate_type_import_code_for_module(self, module, env, code):
3504
        # Generate type import code for all exported extension types in
3505
        # an imported module.
3506
        #if module.c_class_entries:
3507 3508 3509 3510
        with ModuleImportGenerator(code) as import_generator:
            for entry in module.c_class_entries:
                if entry.defined_in_pxd:
                    self.generate_type_import_code(env, entry.type, entry.pos, code, import_generator)
3511

3512
    def specialize_fused_types(self, pxd_env):
3513 3514 3515 3516 3517 3518 3519 3520 3521
        """
        If fused c(p)def functions are defined in an imported pxd, but not
        used in this implementation file, we still have fused entries and
        not specialized ones. This method replaces any fused entries with their
        specialized ones.
        """
        for entry in pxd_env.cfunc_entries[:]:
            if entry.type.is_fused:
                # This call modifies the cfunc_entries in-place
3522
                entry.type.get_all_specialized_function_types()
3523

3524 3525 3526 3527 3528 3529 3530
    def generate_c_variable_import_code_for_module(self, module, env, code):
        # Generate import code for all exported C functions in a cimported module.
        entries = []
        for entry in module.var_entries:
            if entry.defined_in_pxd:
                entries.append(entry)
        if entries:
3531 3532
            env.use_utility_code(
                UtilityCode.load_cached("VoidPtrImport", "ImportExport.c"))
3533 3534
            temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
            code.putln(
3535
                '%s = PyImport_ImportModule("%s"); if (!%s) %s' % (
3536 3537 3538 3539 3540 3541 3542 3543 3544
                    temp,
                    module.qualified_name,
                    temp,
                    code.error_goto(self.pos)))
            for entry in entries:
                if env is module:
                    cname = entry.cname
                else:
                    cname = module.mangle(Naming.varptr_prefix, entry.name)
3545
                signature = entry.type.empty_declaration_code()
3546 3547 3548 3549 3550 3551
                code.putln(
                    'if (__Pyx_ImportVoidPtr(%s, "%s", (void **)&%s, "%s") < 0) %s' % (
                        temp, entry.name, cname, signature,
                        code.error_goto(self.pos)))
            code.putln("Py_DECREF(%s); %s = 0;" % (temp, temp))

3552 3553 3554 3555
    def generate_c_function_import_code_for_module(self, module, env, code):
        # Generate import code for all exported C functions in a cimported module.
        entries = []
        for entry in module.cfunc_entries:
3556
            if entry.defined_in_pxd and entry.used:
3557 3558
                entries.append(entry)
        if entries:
3559 3560
            env.use_utility_code(
                UtilityCode.load_cached("FunctionImport", "ImportExport.c"))
3561
            temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
3562
            code.putln(
3563
                '%s = PyImport_ImportModule("%s"); if (!%s) %s' % (
3564 3565 3566 3567
                    temp,
                    module.qualified_name,
                    temp,
                    code.error_goto(self.pos)))
3568
            for entry in entries:
Mark Florisson's avatar
Mark Florisson committed
3569
                code.putln(
3570
                    'if (__Pyx_ImportFunction(%s, %s, (void (**)(void))&%s, "%s") < 0) %s' % (
Mark Florisson's avatar
Mark Florisson committed
3571
                        temp,
3572
                        entry.name.as_c_string_literal(),
Mark Florisson's avatar
Mark Florisson committed
3573 3574 3575
                        entry.cname,
                        entry.type.signature_string(),
                        code.error_goto(self.pos)))
3576
            code.putln("Py_DECREF(%s); %s = 0;" % (temp, temp))
3577

3578 3579 3580
    def generate_type_init_code(self, env, code):
        # Generate type import code for extern extension types
        # and type ready code for non-extern ones.
3581 3582 3583 3584 3585 3586 3587 3588 3589
        with ModuleImportGenerator(code) as import_generator:
            for entry in env.c_class_entries:
                if entry.visibility == 'extern' and not entry.utility_code_definition:
                    self.generate_type_import_code(env, entry.type, entry.pos, code, import_generator)
                else:
                    self.generate_base_type_import_code(env, entry, code, import_generator)
                    self.generate_exttype_vtable_init_code(entry, code)
                    if entry.type.early_init:
                        self.generate_type_ready_code(entry, code)
3590

3591
    def generate_base_type_import_code(self, env, entry, code, import_generator):
3592
        base_type = entry.type.base_type
3593
        if (base_type and base_type.module_name != env.qualified_name and not
3594 3595
                (base_type.is_builtin_type or base_type.is_cython_builtin_type)
                 and not entry.utility_code_definition):
3596
            self.generate_type_import_code(env, base_type, self.pos, code, import_generator)
3597

3598
    def generate_type_import_code(self, env, type, pos, code, import_generator):
3599 3600 3601 3602 3603
        # If not already done, generate code to import the typeobject of an
        # extension type defined in another module, and extract its C method
        # table pointer if any.
        if type in env.types_imported:
            return
3604 3605 3606 3607
        if type.name not in Code.ctypedef_builtins_map:
            # see corresponding condition in generate_type_import_call() below!
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("TypeImport", "ImportExport.c"))
3608
        self.generate_type_import_call(type, code, import_generator, error_pos=pos)
3609
        if type.vtabptr_cname:
3610
            code.globalstate.use_utility_code(
3611
                UtilityCode.load_cached('GetVTable', 'ImportExport.c'))
3612
            code.putln("%s = (struct %s*)__Pyx_GetVtable(%s); %s" % (
3613 3614 3615 3616
                type.vtabptr_cname,
                type.vtabstruct_cname,
                type.typeptr_cname,
                code.error_goto_if_null(type.vtabptr_cname, pos)))
3617
        env.types_imported.add(type)
3618

3619
    def generate_type_import_call(self, type, code, import_generator, error_code=None, error_pos=None):
3620 3621 3622 3623
        if type.typedef_flag:
            objstruct = type.objstruct_cname
        else:
            objstruct = "struct %s" % type.objstruct_cname
3624
        sizeof_objstruct = objstruct
3625
        module_name = type.module_name
3626
        condition = replacement = None
3627 3628
        if module_name not in ('__builtin__', 'builtins'):
            module_name = '"%s"' % module_name
3629 3630 3631 3632 3633
        elif type.name in Code.ctypedef_builtins_map:
            # Fast path for special builtins, don't actually import
            ctypename = Code.ctypedef_builtins_map[type.name]
            code.putln('%s = %s;' % (type.typeptr_cname, ctypename))
            return
3634 3635
        else:
            module_name = '__Pyx_BUILTIN_MODULE_NAME'
3636
            if type.name in Code.non_portable_builtins_map:
Stefan Behnel's avatar
Stefan Behnel committed
3637
                condition, replacement = Code.non_portable_builtins_map[type.name]
3638 3639 3640 3641
            if objstruct in Code.basicsize_builtins_map:
                # Some builtin types have a tp_basicsize which differs from sizeof(...):
                sizeof_objstruct = Code.basicsize_builtins_map[objstruct]

3642 3643 3644 3645
        if not error_code:
            assert error_pos is not None
            error_code = code.error_goto(error_pos)

3646 3647
        module = import_generator.imported_module(module_name, error_code)
        code.put('%s = __Pyx_ImportType(%s, %s,' % (
3648
            type.typeptr_cname,
3649
            module,
3650 3651
            module_name))

3652 3653
        type_name = type.name.as_c_string_literal()

3654 3655 3656 3657 3658
        if condition and replacement:
            code.putln("")  # start in new line
            code.putln("#if %s" % condition)
            code.putln('"%s",' % replacement)
            code.putln("#else")
3659
            code.putln('%s,' % type_name)
3660 3661
            code.putln("#endif")
        else:
3662
            code.put(' %s, ' % type_name)
3663 3664 3665 3666

        if sizeof_objstruct != objstruct:
            if not condition:
                code.putln("")  # start in new line
3667
            code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000")
3668
            code.putln('sizeof(%s),' % objstruct)
3669 3670
            code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API")
            code.putln('sizeof(%s),' % objstruct)
3671 3672
            code.putln("#else")
            code.putln('sizeof(%s),' % sizeof_objstruct)
3673
            code.putln("#endif")
3674 3675 3676
        else:
            code.put('sizeof(%s), ' % objstruct)

3677
        # check_size
3678 3679 3680 3681
        if type.check_size and type.check_size in ('error', 'warn', 'ignore'):
            check_size = type.check_size
        elif not type.is_external or type.is_subclassed:
            check_size = 'error'
3682
        else:
3683 3684
            raise RuntimeError("invalid value for check_size '%s' when compiling %s.%s" % (
                type.check_size, module_name, type.name))
3685
        code.putln('__Pyx_ImportType_CheckSize_%s);' % check_size.title())
3686 3687

        code.putln(' if (!%s) %s' % (type.typeptr_cname, error_code))
3688

3689 3690
    def generate_type_ready_code(self, entry, code):
        Nodes.CClassDefNode.generate_type_ready_code(entry, code)
3691

da-woods's avatar
da-woods committed
3692 3693 3694 3695
    def is_main_module_flag_cname(self):
        full_module_name = self.full_module_name.replace('.', '__')
        return self.punycode_module_name(Naming.module_is_main, full_module_name)

3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710
    def generate_exttype_vtable_init_code(self, entry, code):
        # Generate code to initialise the C method table of an
        # extension type.
        type = entry.type
        if type.vtable_cname:
            code.putln(
                "%s = &%s;" % (
                    type.vtabptr_cname,
                    type.vtable_cname))
            if type.base_type and type.base_type.vtabptr_cname:
                code.putln(
                    "%s.%s = *%s;" % (
                        type.vtable_cname,
                        Naming.obj_base_cname,
                        type.base_type.vtabptr_cname))
3711 3712 3713

            c_method_entries = [
                entry for entry in type.scope.cfunc_entries
3714
                if entry.func_cname]
3715 3716 3717 3718 3719 3720 3721 3722 3723
            if c_method_entries:
                for meth_entry in c_method_entries:
                    cast = meth_entry.type.signature_cast_string()
                    code.putln(
                        "%s.%s = %s%s;" % (
                            type.vtable_cname,
                            meth_entry.cname,
                            cast,
                            meth_entry.func_cname))
3724

3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745

class ModuleImportGenerator(object):
    """
    Helper to generate module import while importing external types.
    This is used to avoid excessive re-imports of external modules when multiple types are looked up.
    """
    def __init__(self, code, imported_modules=None):
        self.code = code
        self.imported = {}
        if imported_modules:
            for name, cname in imported_modules.items():
                self.imported['"%s"' % name] = cname
        self.temps = []  # remember original import order for freeing

    def imported_module(self, module_name_string, error_code):
        if module_name_string in self.imported:
            return self.imported[module_name_string]

        code = self.code
        temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.temps.append(temp)
3746
        code.putln('%s = PyImport_ImportModule(%s); if (unlikely(!%s)) %s' % (
3747
            temp, module_name_string, temp, error_code))
3748
        code.put_gotref(temp, py_object_type)
3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761
        self.imported[module_name_string] = temp
        return temp

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        code = self.code
        for temp in self.temps:
            code.put_decref_clear(temp, py_object_type)
            code.funcstate.release_temp(temp)


3762
def generate_cfunction_declaration(entry, env, code, definition):
3763
    from_cy_utility = entry.used and entry.utility_code_definition
3764 3765
    if entry.used and entry.inline_func_in_pxd or (not entry.in_cinclude and (
            definition or entry.defined_in_pxd or entry.visibility == 'extern' or from_cy_utility)):
3766
        if entry.visibility == 'extern':
3767
            storage_class = Naming.extern_c_macro
3768 3769
            dll_linkage = "DL_IMPORT"
        elif entry.visibility == 'public':
3770
            storage_class = Naming.extern_c_macro
3771
            dll_linkage = None
3772
        elif entry.visibility == 'private':
3773
            storage_class = "static"
3774 3775
            dll_linkage = None
        else:
3776
            storage_class = "static"
3777 3778 3779 3780
            dll_linkage = None
        type = entry.type

        if entry.defined_in_pxd and not definition:
3781
            storage_class = "static"
3782 3783 3784
            dll_linkage = None
            type = CPtrType(type)

3785
        header = type.declaration_code(
3786
            entry.cname, dll_linkage=dll_linkage)
3787 3788
        modifiers = code.build_function_modifiers(entry.func_modifiers)
        code.putln("%s %s%s; /*proto*/" % (
3789 3790 3791 3792
            storage_class,
            modifiers,
            header))

3793
#------------------------------------------------------------------------------------
Stefan Behnel's avatar
Stefan Behnel committed
3794 3795 3796
#
#  Runtime support code
#
3797 3798
#------------------------------------------------------------------------------------

3799
refnanny_utility_code = UtilityCode.load("Refnanny", "ModuleSetupCode.c")
Robert Bradshaw's avatar
Robert Bradshaw committed
3800

3801 3802 3803 3804 3805 3806
packed_struct_utility_code = UtilityCode(proto="""
#if defined(__GNUC__)
#define __Pyx_PACKED __attribute__((__packed__))
#else
#define __Pyx_PACKED
#endif
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3807
""", impl="", proto_block='utility_code_proto_before_types')
3808

3809
capsule_utility_code = UtilityCode.load("Capsule", "Capsule.c")