some stream-lining in CCodeWriter.mark_pos()

Cython/Compiler/Code.py

@@ -90,13 +90,14 @@ class CCodeWriter:
     def get_py_version_hex(self, pyversion):
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
-    def file_contents(self, source_desc):
+    def commented_file_contents(self, source_desc):
         try:
             return self.input_file_contents[source_desc]
         except KeyError:
-            F = [line.encode('ASCII', 'replace').replace(
-                    '*/', '*[inserted by cython to avoid comment closer]/')
-                 for line in source_desc.get_lines(decode=True)]
+            F = [u' * ' + line.rstrip().replace(
+                    u'*/', u'*[inserted by cython to avoid comment closer]/'
+                    ).encode('ASCII', 'replace') # + Py2 auto-decode to unicode
+                 for line in source_desc.get_lines()]
             self.input_file_contents[source_desc] = F
             return F
 
@@ -105,16 +106,14 @@ class CCodeWriter:
             return
         source_desc, line, col = pos
         assert isinstance(source_desc, SourceDescriptor)
-        contents = self.file_contents(source_desc)
+        contents = self.commented_file_contents(source_desc)
 
-        context = ''
-        for i in range(max(0,line-3), min(line+2, len(contents))):
-            s = contents[i]
-            if i+1 == line:   # line numbers in pyrex start counting up from 1
-                s = s.rstrip() + '             # <<<<<<<<<<<<<< ' + '\n'
-            context += " * " + s
+        lines = contents[max(0,line-3):line] # line numbers start at 1
+        lines[-1] += u'             # <<<<<<<<<<<<<<'
+        lines += contents[line:line+2]
 
-        marker = '"%s":%d\n%s' % (source_desc.get_description().encode('ASCII', 'replace'), line, context)
+        marker = u'"%s":%d\n%s\n' % (
+            source_desc.get_escaped_description(), line, u'\n'.join(lines))
         if self.last_marker != marker:
             self.marker = marker
 

Cython/Compiler/Scanning.py

@@ -209,8 +209,15 @@ class SourceDescriptor:
     """
     A SourceDescriptor should be considered immutable.
     """
+    _escaped_description = None
     def __str__(self):
         assert False # To catch all places where a descriptor is used directly as a filename
+    
+    def get_escaped_description(self):
+        if self._escaped_description is None:
+            self._escaped_description = \
+                self.get_description().encode('ASCII', 'replace').decode("ASCII")
+        return self._escaped_description
 
 class FileSourceDescriptor(SourceDescriptor):
     """
@@ -223,16 +230,8 @@ class FileSourceDescriptor(SourceDescriptor):
     def __init__(self, filename):
         self.filename = filename
     
-    def get_lines(self, decode=False):
-        # decode is True when called from Code.py (which reserializes in a standard way to ASCII),
-        # while decode is False when called from Errors.py.
-        #
-        # Note that if changing Errors.py in this respect, raising errors over wrong encoding
-        # will no longer be able to produce the line where the encoding problem occurs ...
-        if decode:
-            return Utils.open_source_file(self.filename)
-        else:
-            return open(self.filename)
+    def get_lines(self):
+        return Utils.open_source_file(self.filename)
     
     def get_description(self):
         return self.filename
@@ -258,7 +257,7 @@ class StringSourceDescriptor(SourceDescriptor):
         self.name = name
         self.codelines = [x + "\n" for x in code.split("\n")]
     
-    def get_lines(self, decode=False):
+    def get_lines(self):
         return self.codelines
     
     def get_description(self):