make it explicit in the generated source code what is an identifier string...

make it explicit in the generated source code what is an identifier string constant and what is a normal string constant that just happens to get interned

Cython/Compiler/ExprNodes.py

@@ -875,7 +875,8 @@ class NameNode(AtomicExprNode):
         self.type = type
         if entry.is_pyglobal or entry.is_builtin:
             assert type.is_pyobject, "Python global or builtin not a Python object"
-            self.interned_cname = self.entry.interned_cname = env.intern(self.entry.name)
+            self.interned_cname = self.entry.interned_cname = \
+                env.intern_identifier(self.entry.name)
 
     def check_identifier_kind(self):
         #print "NameNode.check_identifier_kind:", self.entry.name ###
@@ -1989,7 +1990,7 @@ class AttributeNode(ExprNode):
         if obj_type.is_pyobject:
             self.type = py_object_type
             self.is_py_attr = 1
-            self.interned_attr_cname = env.intern(self.attribute)
+            self.interned_attr_cname = env.intern_identifier(self.attribute)
         else:
             if not obj_type.is_error:
                 error(self.pos, 

Cython/Compiler/ModuleNode.py

@@ -1390,11 +1390,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     Naming.stringtab_cname)
             for entry in entries:
                 code.putln(
-                    "{&%s, %s, sizeof(%s), %d, %d}," % (
+                    "{&%s, %s, sizeof(%s), %d, %d, %d}," % (
                         entry.pystring_cname,
                         entry.cname,
                         entry.cname,
                         entry.type.is_unicode,
+                        entry.is_identifier,
                         entry.is_interned
                         ))
             code.putln(

Cython/Compiler/Nodes.py

@@ -1084,7 +1084,8 @@ class CFuncDefNode(FuncDefNode):
             self.entry.as_variable = self.py_func.entry
             # Reset scope entry the above cfunction
             env.entries[name] = self.entry
-            self.py_func.interned_attr_cname = env.intern(self.py_func.entry.name)
+            self.py_func.interned_attr_cname = env.intern_identifier(
+                self.py_func.entry.name)
             if not env.is_module_scope or Options.lookup_module_cpdef:
                 self.override = OverrideCheckNode(self.pos, py_func = self.py_func)
                 self.body = StatListNode(self.pos, stats=[self.override, self.body])
@@ -2119,7 +2120,8 @@ class PropertyNode(StatNode):
         entry = env.declare_property(self.name, self.doc, self.pos)
         if entry:
             if self.doc and Options.docstrings:
-                doc_entry = env.get_string_const(self.doc)
+                doc_entry = env.get_string_const(
+                    self.doc, identifier = False)
                 entry.doc_cname = doc_entry.cname
             self.body.analyse_declarations(entry.scope)
 
@@ -3679,7 +3681,8 @@ class FromImportStatNode(StatNode):
         self.item.allocate_temp(env)
         self.interned_items = []
         for name, target in self.items:
-            self.interned_items.append((env.intern(name), target))
+            self.interned_items.append(
+                (env.intern_identifier(name), target))
             target.analyse_target_expression(env, None)
             #target.release_target_temp(env) # was release_temp ?!?
         self.module.release_temp(env)
@@ -3713,7 +3716,7 @@ utility_function_predeclarations = \
 #define INLINE 
 #endif
 
-typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern;} __Pyx_StringTabEntry; /*proto*/
+typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/
 
 """ + """
 
@@ -4295,22 +4298,18 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
 ""","""
 static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
     while (t->p) {
-        if (t->is_unicode) {
         #if PY_MAJOR_VERSION < 3
+        if (t->is_unicode) {
             *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
-            #else
-            if (t->intern) {
+        } else if (t->intern) {
+            *t->p = PyString_InternFromString(t->s);
+        #else  /* Python 3+ has unicode identifiers */
+        if (t->is_identifier || (t->is_unicode && t->intern)) {
             *t->p = PyUnicode_InternFromString(t->s);
-            } else {
+        } else if (t->is_unicode) {
             *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
-            }
         #endif
         } else {
-            #if PY_MAJOR_VERSION < 3
-            if (t->intern)
-                *t->p = PyString_InternFromString(t->s);
-            else
-            #endif
             *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
         }
         if (!*t->p)

Cython/Compiler/Symtab.py

@@ -66,6 +66,7 @@ class Entry:
     # is_inherited     boolean    Is an inherited attribute of an extension type
     # pystring_cname   string     C name of Python version of string literal
     # is_interned      boolean    For string const entries, value is interned
+    # is_identifier    boolean    For string const entries, value is an identifier
     # used             boolean
     # is_special       boolean    Is a special method or property accessor
     #                               of an extension type
@@ -104,6 +105,7 @@ class Entry:
     as_module = None
     is_inherited = 0
     pystring_cname = None
+    is_identifier = 0
     is_interned = 0
     used = 0
     is_special = 0
@@ -186,6 +188,7 @@ class Scope:
         self.cname_to_entry = {}
         self.pow_function_used = 0
         self.string_to_entry = {}
+        self.identifier_to_entry = {}
         self.num_to_entry = {}
         self.obj_to_entry = {}
         self.pystring_entries = []
@@ -203,8 +206,8 @@ class Scope:
     def __str__(self):
         return "<%s %s>" % (self.__class__.__name__, self.qualified_name)
 
-    def intern(self, name):
-        return self.global_scope().intern(name)
+    def intern_identifier(self, name):
+        return self.global_scope().intern_identifier(name)
 
     def qualifying_scope(self):
         return self.parent_scope
@@ -446,14 +449,19 @@ class Scope:
         self.const_entries.append(entry)
         return entry
 
-    def get_string_const(self, value):
+    def get_string_const(self, value, identifier = False):
         # Get entry for string constant. Returns an existing
         # one if possible, otherwise creates a new one.
         genv = self.global_scope()
-        entry = genv.string_to_entry.get(value)
+        if identifier:
+            string_map = genv.identifier_to_entry
+        else:
+            string_map = genv.string_to_entry
+        entry = string_map.get(value)
         if not entry:
             entry = self.add_string_const(value)
-            genv.string_to_entry[value] = entry
+            entry.is_identifier = identifier
+            string_map[value] = entry
         return entry
 
     def add_py_string(self, entry):
@@ -669,6 +677,7 @@ class ModuleScope(Scope):
     # python_include_files [string]           Standard  Python headers to be included
     # include_files        [string]           Other C headers to be included
     # string_to_entry      {string : Entry}   Map string const to entry
+    # identifier_to_entry  {string : Entry}   Map identifier string const to entry
     # context              Context
     # parent_module        Scope              Parent in the import namespace
     # module_entries       {string : Entry}   For cimport statements
@@ -740,8 +749,8 @@ class ModuleScope(Scope):
             entry.is_builtin = 1
         return entry
 
-    def intern(self, name):
-        string_entry = self.get_string_const(name)
+    def intern_identifier(self, name):
+        string_entry = self.get_string_const(name, identifier = True)
         self.add_py_string(string_entry)
         return string_entry.pystring_cname
 
@@ -1238,7 +1247,7 @@ class CClassScope(ClassScope):
                                   # I keep it in for now. is_member should be enough
                                   # later on
             entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
-            entry.interned_cname = self.intern(name)
+            entry.interned_cname = self.intern_identifier(name)
             return entry