- Included STX i18n patch by Andrey Lebedev to provide compatibility

       with utf-8 encoded strings.

doc/CHANGES.txt

@@ -7,6 +7,9 @@ Zope Changes
   HEAD only
 
     Features added
+ 
+     - Included STX i18n patch by Andrey Lebedev to provide compatibility
+       with utf-8 encoded strings.
 
      - Show known medusa servers in Control_Panel.
 

lib/python/StructuredText/DocumentClass.py

@@ -304,7 +304,7 @@ class DocumentClass:
         'doc_xref',
         ]
 
-    def __call__(self, doc):
+    def __call__(self, doc): 
         if type(doc) in StringTypes:
             doc=ST.StructuredText(doc)
             doc.setSubparagraphs(self.color_paragraphs(
@@ -786,7 +786,7 @@ class DocumentClass:
 
     def doc_literal(
         self, s,
-        expr = re.compile(r"(\W+|^)'([%s%s%s\s]+)'([%s]+|$)" % (letters, digits, literal_punc, phrase_delimiters)).search,):
+        expr = re.compile(r"(\W+|^)'((?:\w|[%s%s\s])+)'([%s]+|$)" % (digits, literal_punc, phrase_delimiters), re.U).search,):
 
         # old expr... failed to cross newlines.
         #        expr=re.compile(
@@ -804,7 +804,9 @@ class DocumentClass:
 
     def doc_emphasize(
         self, s,
-        expr = re.compile(r'\*([%s%s%s\s]+?)\*' % (letters, digits, strongem_punc)).search
+        # i18nal variant
+        expr = re.compile(r'\*((?:\w|[%s\s])+?)\*' % (strongem_punc), re.U).search
+        #expr = re.compile(r'\*([%s%s%s\s]+?)\*' % (letters, digits, strongem_punc)).search
         #expr = re.compile(r'\s*\*([ \n\r%s0-9.:/;,\'\"\?\-\_\/\=\-\>\<\(\)]+)\*(?!\*|-)' % letters).search       # old expr, inconsistent punctuation
         ):
 
@@ -850,7 +852,7 @@ class DocumentClass:
 
     def doc_underline(self,
                       s,
-                      expr=re.compile(r'_([%s%s%s\s]+)_([\s%s]|$)' % (letters, digits, under_punc,phrase_delimiters)).search):
+                      expr=re.compile(r'_((?:\w|[%s\s])+)_([\s%s]|$)' % (under_punc,phrase_delimiters), re.U).search):
 
         result = expr(s)
         if result:
@@ -864,7 +866,7 @@ class DocumentClass:
 
     def doc_strong(self,
                    s,
-                   expr = re.compile(r'\*\*([%s%s%s\s]+?)\*\*' % (letters, digits, strongem_punc)).search
+                   expr = re.compile(r'\*\*((?:\w|[%s%s\s])+?)\*\*' % (digits, strongem_punc), re.U).search
                    #expr = re.compile(r'\s*\*\*([ \n\r%s0-9.:/;,\'\"\?\-\_\/\=\-\>\<\(\)]+)\*\*(?!\*|-)' % letters).search, # old expr, inconsistent punc, failed to cross newlines.
         ):
 
@@ -876,7 +878,7 @@ class DocumentClass:
             return None
 
     ## Some constants to make the doc_href() regex easier to read.
-    _DQUOTEDTEXT = r'("[ %s0-9\n\r%s]+")' % (letters,dbl_quoted_punc) ## double quoted text
+    _DQUOTEDTEXT = r'("(?:\w|[ 0-9\n\r%s])+")' % (dbl_quoted_punc) ## double quoted text
     _ABSOLUTE_URL=r'((http|https|ftp|mailto|file|about)[:/]+?[%s0-9_\@\.\,\?\!\/\:\;\-\#\~\=\&\%%\+]+)' % letters
     _ABS_AND_RELATIVE_URL=r'([%s0-9_\@\.\,\?\!\/\:\;\-\#\~\=\&\%%\+]+)' % letters
 
@@ -884,12 +886,12 @@ class DocumentClass:
 
 
     def doc_href1(self, s,
-                  expr=re.compile(_DQUOTEDTEXT + "(:)" + _ABS_AND_RELATIVE_URL + _SPACES).search
+                  expr=re.compile(_DQUOTEDTEXT + "(:)" + _ABS_AND_RELATIVE_URL + _SPACES, re.U).search
                    ):
         return self.doc_href(s, expr)
 
     def doc_href2(self, s,
-                  expr=re.compile(_DQUOTEDTEXT + r'(\,\s+)' + _ABSOLUTE_URL + _SPACES).search
+                  expr=re.compile(_DQUOTEDTEXT + r'(\,\s+)' + _ABSOLUTE_URL + _SPACES, re.U).search
                    ):
         return self.doc_href(s, expr)
 

lib/python/StructuredText/DocumentWithImages.py

@@ -30,13 +30,13 @@ class DocumentWithImages(DocumentClass):
 
     def doc_img(
         self, s,
-        expr1=re.compile('\"([ _a-zA-Z0-9*.:/;,\-\n\~]+)\":img:([a-zA-Z0-9\_\-.:/;,\n\~]+)').search,
-        expr2=re.compile('\"([ _a-zA-Z0-9*.:/;,\-\n\~]+)\":img:([a-zA-Z0-9\_\-.:/;,\n\~]+):([a-zA-Z0-9_\-.:/;,\n\~]+)').search
+        expr1=re.compile('\"((?:\w|[ *.:/;,\-\n\~])+)\":img:([a-zA-Z0-9\_\-.:/;,\n\~]+)', re.U).search,
+        expr2=re.compile('\"((?:\w|[ *.:/;,\-\n\~])+)\":img:([a-zA-Z0-9\_\-.:/;,\n\~]+):([a-zA-Z0-9_\-.:/;,\n\~]+)', re.U).search
         ):
-
         r = expr2(s)
         if r:
 
+
             # Warning: the regex are getting confused when the string after :img:
             # is an URL containing ":" (Collector #2276)
             # Ugly workaround: check if have an absolute URL here. Not a cool solution,

lib/python/StructuredText/ST.py

@@ -115,6 +115,9 @@ def StructuredText(paragraphs, delimiter=re.compile(para_delim)):
     which mimics the structure of the paragraphs.
     Structure => [paragraph,[sub-paragraphs]]
     """
+    if type(paragraphs) == type(''):
+        paragraphs = unicode(paragraphs, 'utf-8')
+
 
     currentlevel    = 0
     currentindent  = 0

lib/python/StructuredText/tests/testStructuredText.py

@@ -210,13 +210,17 @@ class BasicTests(unittest.TestCase):
         '<code>"literal":http://www.zope.org/.</code>')
 
 
-    def XXXtestUnicodeContent(self):
-        # This fails because ST uses the default locale to get "letters"
-        # whereas it should use \w+ and re.U if the string is Unicode.
-        #self._test(u"h\xe9 **y\xe9** xx",
-        #           u"h\xe9 <strong>y\xe9</strong> xx")
-        pass
-
+    def testUnicodeContent(self):
+        self._test(u"h\xe9 **y\xe9** xx",
+                   u"h\xe9 <strong>y\xe9</strong> xx")
+        self._test(u"h\xe9 *y\xe9* xx",
+                   u"h\xe9 <em>y\xe9</em> xx")
+        self._test(u"h\xe9 _y\xe9_ xx",
+                   u"h\xe9 <u>y\xe9</u> xx")
+        self._test(u"h\xe9 'y\xe9' xx",
+                   u"h\xe9 <code>y\xe9</code> xx")
+        self._test(u"h\xe9 \"y\xe9\":http://zope.org xx",
+                   u"h\xe9 <a href=\"http://zope.org\">y\xe9</a> xx")
 
 def test_suite():
     suite = unittest.TestSuite()