Commit 5cb5c685 authored by Stefan Behnel's avatar Stefan Behnel

make it explicit in the generated source code what is an identifier string...

make it explicit in the generated source code what is an identifier string constant and what is a normal string constant that just happens to get interned
parent 84827de7
......@@ -875,7 +875,8 @@ class NameNode(AtomicExprNode):
self.type = type
if entry.is_pyglobal or entry.is_builtin:
assert type.is_pyobject, "Python global or builtin not a Python object"
self.interned_cname = self.entry.interned_cname = env.intern(self.entry.name)
self.interned_cname = self.entry.interned_cname = \
env.intern_identifier(self.entry.name)
def check_identifier_kind(self):
#print "NameNode.check_identifier_kind:", self.entry.name ###
......@@ -1989,7 +1990,7 @@ class AttributeNode(ExprNode):
if obj_type.is_pyobject:
self.type = py_object_type
self.is_py_attr = 1
self.interned_attr_cname = env.intern(self.attribute)
self.interned_attr_cname = env.intern_identifier(self.attribute)
else:
if not obj_type.is_error:
error(self.pos,
......
......@@ -1390,11 +1390,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
Naming.stringtab_cname)
for entry in entries:
code.putln(
"{&%s, %s, sizeof(%s), %d, %d}," % (
"{&%s, %s, sizeof(%s), %d, %d, %d}," % (
entry.pystring_cname,
entry.cname,
entry.cname,
entry.type.is_unicode,
entry.is_identifier,
entry.is_interned
))
code.putln(
......
......@@ -1084,7 +1084,8 @@ class CFuncDefNode(FuncDefNode):
self.entry.as_variable = self.py_func.entry
# Reset scope entry the above cfunction
env.entries[name] = self.entry
self.py_func.interned_attr_cname = env.intern(self.py_func.entry.name)
self.py_func.interned_attr_cname = env.intern_identifier(
self.py_func.entry.name)
if not env.is_module_scope or Options.lookup_module_cpdef:
self.override = OverrideCheckNode(self.pos, py_func = self.py_func)
self.body = StatListNode(self.pos, stats=[self.override, self.body])
......@@ -2119,7 +2120,8 @@ class PropertyNode(StatNode):
entry = env.declare_property(self.name, self.doc, self.pos)
if entry:
if self.doc and Options.docstrings:
doc_entry = env.get_string_const(self.doc)
doc_entry = env.get_string_const(
self.doc, identifier = False)
entry.doc_cname = doc_entry.cname
self.body.analyse_declarations(entry.scope)
......@@ -3679,7 +3681,8 @@ class FromImportStatNode(StatNode):
self.item.allocate_temp(env)
self.interned_items = []
for name, target in self.items:
self.interned_items.append((env.intern(name), target))
self.interned_items.append(
(env.intern_identifier(name), target))
target.analyse_target_expression(env, None)
#target.release_target_temp(env) # was release_temp ?!?
self.module.release_temp(env)
......@@ -3713,7 +3716,7 @@ utility_function_predeclarations = \
#define INLINE
#endif
typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern;} __Pyx_StringTabEntry; /*proto*/
typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/
""" + """
......@@ -4295,22 +4298,18 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
""","""
static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
while (t->p) {
if (t->is_unicode) {
#if PY_MAJOR_VERSION < 3
if (t->is_unicode) {
*t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
#else
if (t->intern) {
} else if (t->intern) {
*t->p = PyString_InternFromString(t->s);
#else /* Python 3+ has unicode identifiers */
if (t->is_identifier || (t->is_unicode && t->intern)) {
*t->p = PyUnicode_InternFromString(t->s);
} else {
} else if (t->is_unicode) {
*t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
}
#endif
} else {
#if PY_MAJOR_VERSION < 3
if (t->intern)
*t->p = PyString_InternFromString(t->s);
else
#endif
*t->p = PyString_FromStringAndSize(t->s, t->n - 1);
}
if (!*t->p)
......
......@@ -66,6 +66,7 @@ class Entry:
# is_inherited boolean Is an inherited attribute of an extension type
# pystring_cname string C name of Python version of string literal
# is_interned boolean For string const entries, value is interned
# is_identifier boolean For string const entries, value is an identifier
# used boolean
# is_special boolean Is a special method or property accessor
# of an extension type
......@@ -104,6 +105,7 @@ class Entry:
as_module = None
is_inherited = 0
pystring_cname = None
is_identifier = 0
is_interned = 0
used = 0
is_special = 0
......@@ -186,6 +188,7 @@ class Scope:
self.cname_to_entry = {}
self.pow_function_used = 0
self.string_to_entry = {}
self.identifier_to_entry = {}
self.num_to_entry = {}
self.obj_to_entry = {}
self.pystring_entries = []
......@@ -203,8 +206,8 @@ class Scope:
def __str__(self):
return "<%s %s>" % (self.__class__.__name__, self.qualified_name)
def intern(self, name):
return self.global_scope().intern(name)
def intern_identifier(self, name):
return self.global_scope().intern_identifier(name)
def qualifying_scope(self):
return self.parent_scope
......@@ -446,14 +449,19 @@ class Scope:
self.const_entries.append(entry)
return entry
def get_string_const(self, value):
def get_string_const(self, value, identifier = False):
# Get entry for string constant. Returns an existing
# one if possible, otherwise creates a new one.
genv = self.global_scope()
entry = genv.string_to_entry.get(value)
if identifier:
string_map = genv.identifier_to_entry
else:
string_map = genv.string_to_entry
entry = string_map.get(value)
if not entry:
entry = self.add_string_const(value)
genv.string_to_entry[value] = entry
entry.is_identifier = identifier
string_map[value] = entry
return entry
def add_py_string(self, entry):
......@@ -669,6 +677,7 @@ class ModuleScope(Scope):
# python_include_files [string] Standard Python headers to be included
# include_files [string] Other C headers to be included
# string_to_entry {string : Entry} Map string const to entry
# identifier_to_entry {string : Entry} Map identifier string const to entry
# context Context
# parent_module Scope Parent in the import namespace
# module_entries {string : Entry} For cimport statements
......@@ -740,8 +749,8 @@ class ModuleScope(Scope):
entry.is_builtin = 1
return entry
def intern(self, name):
string_entry = self.get_string_const(name)
def intern_identifier(self, name):
string_entry = self.get_string_const(name, identifier = True)
self.add_py_string(string_entry)
return string_entry.pystring_cname
......@@ -1238,7 +1247,7 @@ class CClassScope(ClassScope):
# I keep it in for now. is_member should be enough
# later on
entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
entry.interned_cname = self.intern(name)
entry.interned_cname = self.intern_identifier(name)
return entry
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment