Commit 5cb5c685 authored by Stefan Behnel's avatar Stefan Behnel

make it explicit in the generated source code what is an identifier string...

make it explicit in the generated source code what is an identifier string constant and what is a normal string constant that just happens to get interned
parent 84827de7
...@@ -875,7 +875,8 @@ class NameNode(AtomicExprNode): ...@@ -875,7 +875,8 @@ class NameNode(AtomicExprNode):
self.type = type self.type = type
if entry.is_pyglobal or entry.is_builtin: if entry.is_pyglobal or entry.is_builtin:
assert type.is_pyobject, "Python global or builtin not a Python object" assert type.is_pyobject, "Python global or builtin not a Python object"
self.interned_cname = self.entry.interned_cname = env.intern(self.entry.name) self.interned_cname = self.entry.interned_cname = \
env.intern_identifier(self.entry.name)
def check_identifier_kind(self): def check_identifier_kind(self):
#print "NameNode.check_identifier_kind:", self.entry.name ### #print "NameNode.check_identifier_kind:", self.entry.name ###
...@@ -1989,7 +1990,7 @@ class AttributeNode(ExprNode): ...@@ -1989,7 +1990,7 @@ class AttributeNode(ExprNode):
if obj_type.is_pyobject: if obj_type.is_pyobject:
self.type = py_object_type self.type = py_object_type
self.is_py_attr = 1 self.is_py_attr = 1
self.interned_attr_cname = env.intern(self.attribute) self.interned_attr_cname = env.intern_identifier(self.attribute)
else: else:
if not obj_type.is_error: if not obj_type.is_error:
error(self.pos, error(self.pos,
......
...@@ -1390,11 +1390,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -1390,11 +1390,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
Naming.stringtab_cname) Naming.stringtab_cname)
for entry in entries: for entry in entries:
code.putln( code.putln(
"{&%s, %s, sizeof(%s), %d, %d}," % ( "{&%s, %s, sizeof(%s), %d, %d, %d}," % (
entry.pystring_cname, entry.pystring_cname,
entry.cname, entry.cname,
entry.cname, entry.cname,
entry.type.is_unicode, entry.type.is_unicode,
entry.is_identifier,
entry.is_interned entry.is_interned
)) ))
code.putln( code.putln(
......
...@@ -1084,7 +1084,8 @@ class CFuncDefNode(FuncDefNode): ...@@ -1084,7 +1084,8 @@ class CFuncDefNode(FuncDefNode):
self.entry.as_variable = self.py_func.entry self.entry.as_variable = self.py_func.entry
# Reset scope entry the above cfunction # Reset scope entry the above cfunction
env.entries[name] = self.entry env.entries[name] = self.entry
self.py_func.interned_attr_cname = env.intern(self.py_func.entry.name) self.py_func.interned_attr_cname = env.intern_identifier(
self.py_func.entry.name)
if not env.is_module_scope or Options.lookup_module_cpdef: if not env.is_module_scope or Options.lookup_module_cpdef:
self.override = OverrideCheckNode(self.pos, py_func = self.py_func) self.override = OverrideCheckNode(self.pos, py_func = self.py_func)
self.body = StatListNode(self.pos, stats=[self.override, self.body]) self.body = StatListNode(self.pos, stats=[self.override, self.body])
...@@ -2119,10 +2120,11 @@ class PropertyNode(StatNode): ...@@ -2119,10 +2120,11 @@ class PropertyNode(StatNode):
entry = env.declare_property(self.name, self.doc, self.pos) entry = env.declare_property(self.name, self.doc, self.pos)
if entry: if entry:
if self.doc and Options.docstrings: if self.doc and Options.docstrings:
doc_entry = env.get_string_const(self.doc) doc_entry = env.get_string_const(
self.doc, identifier = False)
entry.doc_cname = doc_entry.cname entry.doc_cname = doc_entry.cname
self.body.analyse_declarations(entry.scope) self.body.analyse_declarations(entry.scope)
def analyse_expressions(self, env): def analyse_expressions(self, env):
self.body.analyse_expressions(env) self.body.analyse_expressions(env)
...@@ -3679,7 +3681,8 @@ class FromImportStatNode(StatNode): ...@@ -3679,7 +3681,8 @@ class FromImportStatNode(StatNode):
self.item.allocate_temp(env) self.item.allocate_temp(env)
self.interned_items = [] self.interned_items = []
for name, target in self.items: for name, target in self.items:
self.interned_items.append((env.intern(name), target)) self.interned_items.append(
(env.intern_identifier(name), target))
target.analyse_target_expression(env, None) target.analyse_target_expression(env, None)
#target.release_target_temp(env) # was release_temp ?!? #target.release_target_temp(env) # was release_temp ?!?
self.module.release_temp(env) self.module.release_temp(env)
...@@ -3713,7 +3716,7 @@ utility_function_predeclarations = \ ...@@ -3713,7 +3716,7 @@ utility_function_predeclarations = \
#define INLINE #define INLINE
#endif #endif
typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern;} __Pyx_StringTabEntry; /*proto*/ typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/
""" + """ """ + """
...@@ -4295,23 +4298,19 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ ...@@ -4295,23 +4298,19 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
""",""" ""","""
static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
while (t->p) { while (t->p) {
#if PY_MAJOR_VERSION < 3
if (t->is_unicode) { if (t->is_unicode) {
#if PY_MAJOR_VERSION < 3
*t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
#else } else if (t->intern) {
if (t->intern) { *t->p = PyString_InternFromString(t->s);
*t->p = PyUnicode_InternFromString(t->s); #else /* Python 3+ has unicode identifiers */
} else { if (t->is_identifier || (t->is_unicode && t->intern)) {
*t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); *t->p = PyUnicode_InternFromString(t->s);
} } else if (t->is_unicode) {
#endif *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
#endif
} else { } else {
#if PY_MAJOR_VERSION < 3 *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
if (t->intern)
*t->p = PyString_InternFromString(t->s);
else
#endif
*t->p = PyString_FromStringAndSize(t->s, t->n - 1);
} }
if (!*t->p) if (!*t->p)
return -1; return -1;
......
...@@ -66,6 +66,7 @@ class Entry: ...@@ -66,6 +66,7 @@ class Entry:
# is_inherited boolean Is an inherited attribute of an extension type # is_inherited boolean Is an inherited attribute of an extension type
# pystring_cname string C name of Python version of string literal # pystring_cname string C name of Python version of string literal
# is_interned boolean For string const entries, value is interned # is_interned boolean For string const entries, value is interned
# is_identifier boolean For string const entries, value is an identifier
# used boolean # used boolean
# is_special boolean Is a special method or property accessor # is_special boolean Is a special method or property accessor
# of an extension type # of an extension type
...@@ -104,6 +105,7 @@ class Entry: ...@@ -104,6 +105,7 @@ class Entry:
as_module = None as_module = None
is_inherited = 0 is_inherited = 0
pystring_cname = None pystring_cname = None
is_identifier = 0
is_interned = 0 is_interned = 0
used = 0 used = 0
is_special = 0 is_special = 0
...@@ -186,6 +188,7 @@ class Scope: ...@@ -186,6 +188,7 @@ class Scope:
self.cname_to_entry = {} self.cname_to_entry = {}
self.pow_function_used = 0 self.pow_function_used = 0
self.string_to_entry = {} self.string_to_entry = {}
self.identifier_to_entry = {}
self.num_to_entry = {} self.num_to_entry = {}
self.obj_to_entry = {} self.obj_to_entry = {}
self.pystring_entries = [] self.pystring_entries = []
...@@ -203,8 +206,8 @@ class Scope: ...@@ -203,8 +206,8 @@ class Scope:
def __str__(self): def __str__(self):
return "<%s %s>" % (self.__class__.__name__, self.qualified_name) return "<%s %s>" % (self.__class__.__name__, self.qualified_name)
def intern(self, name): def intern_identifier(self, name):
return self.global_scope().intern(name) return self.global_scope().intern_identifier(name)
def qualifying_scope(self): def qualifying_scope(self):
return self.parent_scope return self.parent_scope
...@@ -446,14 +449,19 @@ class Scope: ...@@ -446,14 +449,19 @@ class Scope:
self.const_entries.append(entry) self.const_entries.append(entry)
return entry return entry
def get_string_const(self, value): def get_string_const(self, value, identifier = False):
# Get entry for string constant. Returns an existing # Get entry for string constant. Returns an existing
# one if possible, otherwise creates a new one. # one if possible, otherwise creates a new one.
genv = self.global_scope() genv = self.global_scope()
entry = genv.string_to_entry.get(value) if identifier:
string_map = genv.identifier_to_entry
else:
string_map = genv.string_to_entry
entry = string_map.get(value)
if not entry: if not entry:
entry = self.add_string_const(value) entry = self.add_string_const(value)
genv.string_to_entry[value] = entry entry.is_identifier = identifier
string_map[value] = entry
return entry return entry
def add_py_string(self, entry): def add_py_string(self, entry):
...@@ -669,6 +677,7 @@ class ModuleScope(Scope): ...@@ -669,6 +677,7 @@ class ModuleScope(Scope):
# python_include_files [string] Standard Python headers to be included # python_include_files [string] Standard Python headers to be included
# include_files [string] Other C headers to be included # include_files [string] Other C headers to be included
# string_to_entry {string : Entry} Map string const to entry # string_to_entry {string : Entry} Map string const to entry
# identifier_to_entry {string : Entry} Map identifier string const to entry
# context Context # context Context
# parent_module Scope Parent in the import namespace # parent_module Scope Parent in the import namespace
# module_entries {string : Entry} For cimport statements # module_entries {string : Entry} For cimport statements
...@@ -740,8 +749,8 @@ class ModuleScope(Scope): ...@@ -740,8 +749,8 @@ class ModuleScope(Scope):
entry.is_builtin = 1 entry.is_builtin = 1
return entry return entry
def intern(self, name): def intern_identifier(self, name):
string_entry = self.get_string_const(name) string_entry = self.get_string_const(name, identifier = True)
self.add_py_string(string_entry) self.add_py_string(string_entry)
return string_entry.pystring_cname return string_entry.pystring_cname
...@@ -1238,7 +1247,7 @@ class CClassScope(ClassScope): ...@@ -1238,7 +1247,7 @@ class CClassScope(ClassScope):
# I keep it in for now. is_member should be enough # I keep it in for now. is_member should be enough
# later on # later on
entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
entry.interned_cname = self.intern(name) entry.interned_cname = self.intern_identifier(name)
return entry return entry
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment