Support utility code in headers (GH-3779)

Specifically this allows public packed structs but may also allow other public declarations that require small amounts of utility code.

Cython/Compiler/Code.py

@@ -1145,6 +1145,14 @@ class GlobalState(object):
         'end'
     ]
 
+    # h files can only have a much smaller list of sections
+    h_code_layout = [
+        'h_code',
+        'utility_code_proto_before_types',
+        'type_declarations',
+        'utility_code_proto',
+        'end'
+    ]
 
     def __init__(self, writer, module_node, code_config, common_utility_include_dir=None):
         self.filename_table = {}
@@ -1215,6 +1223,11 @@ class GlobalState(object):
         code.putln("")
         code.putln("/* --- Runtime support code --- */")
 
+    def initialize_main_h_code(self):
+        rootwriter = self.rootwriter
+        for part in self.h_code_layout:
+            self.parts[part] = rootwriter.insertion_point()
+
     def finalize_main_c_code(self):
         self.close_global_decls()
 

Cython/Compiler/ModuleNode.py

@@ -191,92 +191,94 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         h_extension_types = h_entries(env.c_class_entries)
         if h_types or h_vars or h_funcs or h_extension_types:
             result.h_file = replace_suffix_encoded(result.c_file, ".h")
-            h_code = Code.CCodeWriter()
+            h_code_writer = Code.CCodeWriter()
             c_code_config = generate_c_code_config(env, options)
-            Code.GlobalState(h_code, self, c_code_config)
+            globalstate = Code.GlobalState(h_code_writer, self, c_code_config)
+            globalstate.initialize_main_h_code()  # in-case utility code is used in the header
+            h_code_start = globalstate.parts['h_code']
+            h_code_main = globalstate.parts['type_declarations']
+            h_code_end = globalstate.parts['end']
             if options.generate_pxi:
                 result.i_file = replace_suffix_encoded(result.c_file, ".pxi")
                 i_code = Code.PyrexCodeWriter(result.i_file)
             else:
                 i_code = None
 
-            h_code.put_generated_by()
+            h_code_start.put_generated_by()
             h_guard = self.api_name(Naming.h_guard_prefix, env)
-            h_code.put_h_guard(h_guard)
-            h_code.putln("")
-            h_code.putln('#include "Python.h"')
-            self.generate_type_header_code(h_types, h_code)
+            h_code_start.put_h_guard(h_guard)
+            h_code_start.putln("")
+            h_code_start.putln('#include "Python.h"')
+            self.generate_type_header_code(h_types, h_code_start)
             if options.capi_reexport_cincludes:
-                self.generate_includes(env, [], h_code)
-            h_code.putln("")
+                self.generate_includes(env, [], h_code_start)
+            h_code_start.putln("")
             api_guard = self.api_name(Naming.api_guard_prefix, env)
-            h_code.putln("#ifndef %s" % api_guard)
-            h_code.putln("")
-            self.generate_extern_c_macro_definition(h_code)
-            h_code.putln("")
-            self.generate_dl_import_macro(h_code)
+            h_code_start.putln("#ifndef %s" % api_guard)
+            h_code_start.putln("")
+            self.generate_extern_c_macro_definition(h_code_start)
+            h_code_start.putln("")
+            self.generate_dl_import_macro(h_code_start)
             if h_extension_types:
-                h_code.putln("")
+                h_code_main.putln("")
                 for entry in h_extension_types:
-                    self.generate_cclass_header_code(entry.type, h_code)
+                    self.generate_cclass_header_code(entry.type, h_code_main)
                     if i_code:
                         self.generate_cclass_include_code(entry.type, i_code)
             if h_funcs:
-                h_code.putln("")
+                h_code_main.putln("")
                 for entry in h_funcs:
-                    self.generate_public_declaration(entry, h_code, i_code)
+                    self.generate_public_declaration(entry, h_code_main, i_code)
             if h_vars:
-                h_code.putln("")
+                h_code_main.putln("")
                 for entry in h_vars:
-                    self.generate_public_declaration(entry, h_code, i_code)
-            h_code.putln("")
-            h_code.putln("#endif /* !%s */" % api_guard)
-            h_code.putln("")
-            h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */")
-            h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */")
-            h_code.putln("")
-            h_code.putln("#if PY_MAJOR_VERSION < 3")
+                    self.generate_public_declaration(entry, h_code_main, i_code)
+            h_code_main.putln("")
+            h_code_main.putln("#endif /* !%s */" % api_guard)
+            h_code_main.putln("")
+            h_code_main.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */")
+            h_code_main.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */")
+            h_code_main.putln("")
+            h_code_main.putln("#if PY_MAJOR_VERSION < 3")
             if env.module_name.isascii():
                 py2_mod_name = env.module_name
             else:
                 py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8")
-                h_code.putln('#error "Unicode module names are not supported in Python 2";')
-            h_code.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name)
-            h_code.putln("#else")
+                h_code_main.putln('#error "Unicode module names are not supported in Python 2";')
+            h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name)
+            h_code_main.putln("#else")
             py3_mod_func_name = self.mod_init_func_cname('PyInit', env)
             warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % (
                 py2_mod_name, py3_mod_func_name, py3_mod_func_name))
-            h_code.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.'))
-            h_code.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name)
-            h_code.putln("")
-            h_code.putln("#if PY_VERSION_HEX >= 0x03050000 "
+            h_code_main.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.'))
+            h_code_main.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name)
+            h_code_main.putln("")
+            h_code_main.putln("#if PY_VERSION_HEX >= 0x03050000 "
                 "&& (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) "
                 "|| (defined(__cplusplus) && __cplusplus >= 201402L))")
-            h_code.putln("#if defined(__cplusplus) && __cplusplus >= 201402L")
-            h_code.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal())
-            h_code.putln("#elif defined(__GNUC__) || defined(__clang__)")
-            h_code.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % (
+            h_code_main.putln("#if defined(__cplusplus) && __cplusplus >= 201402L")
+            h_code_main.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal())
+            h_code_main.putln("#elif defined(__GNUC__) || defined(__clang__)")
+            h_code_main.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % (
                 warning_string.as_c_string_literal()))
-            h_code.putln("#elif defined(_MSC_VER)")
-            h_code.putln('__declspec(deprecated(%s)) __inline' % (
+            h_code_main.putln("#elif defined(_MSC_VER)")
+            h_code_main.putln('__declspec(deprecated(%s)) __inline' % (
                 warning_string.as_c_string_literal()))
-            h_code.putln('#endif')
-            h_code.putln("static PyObject* __PYX_WARN_IF_INIT_CALLED(PyObject* res) {")
-            h_code.putln("return res;")
-            h_code.putln("}")
+            h_code_main.putln('#endif')
+            h_code_main.putln("static PyObject* __PYX_WARN_IF_INIT_CALLED(PyObject* res) {")
+            h_code_main.putln("return res;")
+            h_code_main.putln("}")
             # Function call is converted to warning macro; uncalled (pointer) is not
-            h_code.putln('#define %s() __PYX_WARN_IF_INIT_CALLED(%s())' % (
+            h_code_main.putln('#define %s() __PYX_WARN_IF_INIT_CALLED(%s())' % (
                 py3_mod_func_name, py3_mod_func_name))
-            h_code.putln('#endif')
-            h_code.putln('#endif')
-            h_code.putln("")
-            h_code.putln("#endif /* !%s */" % h_guard)
+            h_code_main.putln('#endif')
+            h_code_main.putln('#endif')
 
-            f = open_new_file(result.h_file)
-            try:
-                h_code.copyto(f)
-            finally:
-                f.close()
+            h_code_end.putln("")
+            h_code_end.putln("#endif /* !%s */" % h_guard)
+
+            with open_new_file(result.h_file) as f:
+                h_code_writer.copyto(f)
 
     def generate_public_declaration(self, entry, h_code, i_code):
         h_code.putln("%s %s;" % (

tests/compile/extern_packed_struct.pyx → tests/compile/packed_structs.pyx

@@ -3,3 +3,8 @@
 cdef extern from *:
     cdef packed struct MyStruct:
         char a
+
+cdef public packed struct PublicStruct:
+    int a
+    unsigned char b
+    int c