Commit c901c411 authored by Kirill Smelkov's avatar Kirill Smelkov

jinja2: Handle non-ascii input/output

Currently slapos.recipe.template:jinja2 fails if source contains
non-ascii input:

    <BLANKLINE>
    An internal error occurred due to a bug in either zc.buildout or in a
    recipe being used:
    Traceback (most recent call last):
      File ".../site-packages/zc/buildout/buildout.py", line 1992, in main
        getattr(buildout, command)(args)
      File ".../site-packages/zc/buildout/buildout.py", line 550, in install
        [self[part]['recipe'] for part in install_parts]
      File ".../site-packages/zc/buildout/buildout.py", line 1135, in __getitem__
        options._initialize()
      File ".../site-packages/zc/buildout/buildout.py", line 1239, in _initialize
        self.initialize()
      File ".../site-packages/zc/buildout/buildout.py", line 1248, in initialize
        self.recipe = recipe_class(buildout, name, self)
      File ".../slapos.recipe.template/slapos/recipe/template/jinja2_template.py", line 208, in __init__
        env.compile(source, filename=template),
      File ".../site-packages/jinja2/environment.py", line 551, in compile
        source = self._parse(source, name, filename)
      File ".../site-packages/jinja2/environment.py", line 470, in _parse
        return Parser(self, source, name, encode_filename(filename)).parse()
      File ".../site-packages/jinja2/parser.py", line 31, in __init__
        self.stream = environment._tokenize(source, name, filename, state)
      File ".../site-packages/jinja2/environment.py", line 501, in _tokenize
        source = self.preprocess(source, name, filename)
      File ".../site-packages/jinja2/environment.py", line 495, in preprocess
        self.iter_extensions(), text_type(source))
    UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in position 219: ordinal not in range(128)

it also fails writing-out non-ascii input (which could originate even
programmatically even if source was purely ascii):

    <BLANKLINE>
    An internal error occurred due to a bug in either zc.buildout or in a
    recipe being used:
    Traceback (most recent call last):
      File ".../site-packages/zc/buildout/buildout.py", line 1992, in main
        getattr(buildout, command)(args)
      File ".../site-packages/zc/buildout/buildout.py", line 666, in install
        installed_files = self[part]._call(recipe.install)
      File ".../site-packages/zc/buildout/buildout.py", line 1407, in _call
        return f()
      File ".../slapos/slapos.recipe.template/slapos/recipe/template/jinja2_template.py", line 227, in install
        out.write(self.template.render(**self.context))
    UnicodeEncodeError: 'ascii' codec can't encode characters in position 206-211: ordinal not in range(128)

Fix it.

@vpelletier suggested to add `encoding` option so it is flexible and
'utf-8' is not hardcoded.

/reviewed-by @vpelletier  (on nexedi/slapos.recipe.template!1)
parent f9db590e
...@@ -39,7 +39,9 @@ And according Jinja2 template (kept simple, control structures are possible):: ...@@ -39,7 +39,9 @@ And according Jinja2 template (kept simple, control structures are possible)::
... '${foo:{{bar}}}\n' ... '${foo:{{bar}}}\n'
... 'swallow: {{ json_module.dumps(("african", "european")) }}\n' ... 'swallow: {{ json_module.dumps(("african", "european")) }}\n'
... 'parameters from section: {{ param_dict | dictsort }}\n' ... 'parameters from section: {{ param_dict | dictsort }}\n'
... 'Rendered with {{recipe}}' ... 'Rendered with {{recipe}}\n'
... 'UTF-8 text: привет мир!\n'
... 'Unicode text: {{ "你好世界" }}\n'
... ) ... )
We run buildout:: We run buildout::
...@@ -57,6 +59,8 @@ And the template has been rendered:: ...@@ -57,6 +59,8 @@ And the template has been rendered::
swallow: ["african", "european"] swallow: ["african", "european"]
parameters from section: [('bar', 'bar'), ('foo', '1')] parameters from section: [('bar', 'bar'), ('foo', '1')]
Rendered with slapos.recipe.template:jinja2 Rendered with slapos.recipe.template:jinja2
UTF-8 text: привет мир!
Unicode text: 你好世界
Parameters Parameters
---------- ----------
...@@ -135,6 +139,10 @@ Optional: ...@@ -135,6 +139,10 @@ Optional:
``folder`` ``folder``
Indirect path of a folder. Any file in such folder can be imported. Indirect path of a folder. Any file in such folder can be imported.
``encoding``
Encoding for input template and output file.
Defaults to ``utf-8``.
FAQ FAQ
--- ---
......
...@@ -156,6 +156,8 @@ class Recipe(object): ...@@ -156,6 +156,8 @@ class Recipe(object):
md5sum=options.get('md5sum'), md5sum=options.get('md5sum'),
)[0] )[0]
source = open(template).read() source = open(template).read()
self.encoding = options.get('encoding', 'utf-8')
source = source.decode(self.encoding)
import_delimiter = options.get('import-delimiter', import_delimiter = options.get('import-delimiter',
DEFAULT_IMPORT_DELIMITER) DEFAULT_IMPORT_DELIMITER)
import_dict = {} import_dict = {}
...@@ -223,7 +225,8 @@ class Recipe(object): ...@@ -223,7 +225,8 @@ class Recipe(object):
with os.fdopen(os.open(self.rendered, with os.fdopen(os.open(self.rendered,
os.O_CREAT | os.O_EXCL | os.O_WRONLY, os.O_CREAT | os.O_EXCL | os.O_WRONLY,
self.mode), 'w') as out: self.mode), 'w') as out:
out.write(self.template.render(**self.context)) out.write(self.template.render(**self.context)
.encode(self.encoding))
return self.rendered return self.rendered
update = install update = install
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment