Commit c06cbf38 authored by Arnaud Fontaine's avatar Arnaud Fontaine

py3: Update Shared.DC.ZRDB.{sqltest,sqlvar} monkey patches.

This supports both python2 and python3. For python3, this decode() properly
arguments returned by string_literal() to have str(), otherwise the query string
is an str() (DocumentTemplate) and the arguments are bytes().

In other places, bytes() is used directly but in this case this is not needed
and would require monkey patching (at least) DocumentTemplate.
parent 2e1359b0
Pipeline #20920 failed with stage
in 0 seconds
......@@ -13,57 +13,64 @@
from Shared.DC.ZRDB.sqltest import *
from Shared.DC.ZRDB import sqltest
from DateTime import DateTime
from types import StringType
list_type_list = list, tuple, set, frozenset, dict
if 1: # For easy diff with original
if 1: # For easy diff with original (ZSQLMethods 3.14)
def render(self, md):
name=self.__name__
name = self.__name__
t=self.type
args=self.args
t = self.type
args = self.args
try:
expr=self.expr
if type(expr) is type(''):
v=md[expr]
expr = self.expr
if isinstance(expr, StringTypes):
v = md[expr]
else:
v=expr(md)
v = expr(md)
except (KeyError, NameError):
if args.has_key('optional') and args['optional']:
if 'optional' in args and args['optional']:
return ''
raise ValueError('Missing input variable, <em>%s</em>' % name)
# PATCH: use isinstance instead of type comparison, to allow
# subclassing.
if isinstance(v, list_type_list):
if len(v) > 1 and not self.multiple:
raise ValueError(
'multiple values are not allowed for <em>%s</em>'
% name)
else: v=[v]
msg = 'multiple values are not allowed for <em>%s</em>' % name
raise ValueError(msg)
else:
v = [v]
vs=[]
vs = []
for v in v:
if not v and type(v) is StringType and t != 'string': continue
if t=='int':
if not v and isinstance(v, StringTypes) and t != 'string':
continue
if t == 'int':
try:
if type(v) is StringType:
if v[-1:]=='L':
v=v[:-1]
if isinstance(v, StringTypes):
if six.PY3 and isinstance(v, bytes):
v = v.decode(self.encoding or 'UTF-8')
if v[-1:] == 'L':
v = v[:-1]
int(v)
else: v=str(int(v))
except ValueError:
raise ValueError(
'Invalid integer value for <em>%s</em>' % name)
elif t=='float':
if not v and type(v) is StringType: continue
else:
v = str(int(v))
except (TypeError, ValueError):
msg = 'Invalid integer value for <em>%s</em>' % name
raise ValueError(msg)
elif t == 'float':
if not v and isinstance(v, StringTypes):
continue
try:
if type(v) is StringType: float(v)
else: v=str(float(v))
except ValueError:
raise ValueError(
'Invalid floating-point value for <em>%s</em>' % name)
if six.PY3 and isinstance(v, bytes):
v = v.decode(self.encoding or 'UTF-8')
if isinstance(v, StringTypes):
float(v)
else:
v = str(float(v))
except (TypeError, ValueError):
msg = 'Invalid floating-point value for <em>%s</em>' % name
raise ValueError(msg)
elif t.startswith('datetime'):
# For subsecond precision, use 'datetime(N)' MySQL type,
# where N is the number of digits after the decimal point.
......@@ -73,38 +80,45 @@ if 1: # For easy diff with original
('.%06u' % (v.micros() % 1000000))[:1+n] if n else '')
else:
if not isinstance(v, (str, unicode)):
if not isinstance(v, StringTypes):
v = str(v)
v=md.getitem('sql_quote__',0)(v)
#if find(v,"\'") >= 0: v=join(split(v,"\'"),"''")
#v="'%s'" % v
if isinstance(v, six.binary_type):
v = v.decode('utf-8')
# The call to sql_quote__ can return something that is not
# a native string anymore!
v = md.getitem('sql_quote__', 0)(v)
if isinstance(v, six.binary_type):
v = v.decode('utf-8')
# if v.find("\'") >= 0: v="''".(v.split("\'"))
# v="'%s'" % v
vs.append(v)
if not vs and t=='nb':
if args.has_key('optional') and args['optional']:
if not vs and t == 'nb':
if 'optional' in args and args['optional']:
return ''
else:
raise ValueError(
'Invalid empty string value for <em>%s</em>' % name)
err = 'Invalid empty string value for <em>%s</em>' % name
raise ValueError(err)
if not vs:
if self.optional: return ''
raise ValueError(
'No input was provided for <em>%s</em>' % name)
if self.optional:
return ''
raise ValueError('No input was provided for <em>%s</em>' % name)
if len(vs) > 1:
vs = ', '.join(map(str, vs))
if self.op == '<>':
## Do the equivalent of 'not-equal' for a list,
## "a not in (b,c)"
return "%s not in (%s)" % (self.column, vs)
# Do the equivalent of 'not-equal' for a list,
# "a not in (b,c)"
return '%s not in (%s)' % (self.column, vs)
else:
## "a in (b,c)"
return "%s in (%s)" % (self.column, vs)
return "%s %s %s" % (self.column, self.op, vs[0])
# "a in (b,c)"
return '%s in (%s)' % (self.column, vs)
return '%s %s %s' % (self.column, self.op, vs[0])
SQLTest.render = SQLTest.__call__ = render
new_valid_types = (('int', 'float', 'string', 'nb', 'datetime') + tuple('datetime(%s)' % x for x in xrange(7)))
from builtins import range
new_valid_types = (('int', 'float', 'string', 'nb', 'datetime') + tuple('datetime(%s)' % x for x in range(7)))
try:
# BBB
......
......@@ -14,84 +14,99 @@
# dtml-sqlvar patch to convert None to NULL, and deal with DateTime
from Shared.DC.ZRDB.sqlvar import SQLVar
from Shared.DC.ZRDB.sqlvar import *
from Shared.DC.ZRDB import sqlvar
from string import atoi,atof
from DateTime import DateTime
def SQLVar_render(self, md):
args=self.args
t=args['type']
try:
expr=self.expr
if type(expr) is str: v=md[expr]
else: v=expr(md)
except Exception:
if args.get('optional'):
if 1: # For easy diff with original (ZSQLMethods 3.14)
def render(self, md):
name = self.__name__
args = self.args
t = args['type']
try:
expr = self.expr
if isinstance(expr, StringTypes):
v = md[expr]
else:
v = expr(md)
except Exception:
if 'optional' in args and args['optional']:
return 'null'
if not isinstance(expr, StringTypes):
raise
raise ValueError('Missing input variable, <em>%s</em>' % name)
if v is None:
  • @arnau here I think we have a different behavior, for something like:

    <dtml-sqlvar expr="v" type="string">

    when v is None.

    Before it was rendered as "None", now it is rendered as null.

    I'm not sure what is correct, while looking at this I discovered that we have some cases where we accidentally saved "None" in some columns in a customer project ( this expression was used in a sql method used to insert in datawarehouse ).

    I feel that maybe this case should raise something like ValueError('Invalid None value for <em>%s</em>' % name') and be rendered as null when using optional, like:

    <dtml-sqlvar expr="v" type="string" optional>

    I'm not sure, but I feel the previous behavior was not good and the new behavior is not good either, but in a different way.

    We could also change to just keep the same behavior.

    Edited by Jérome Perrin
  • According to Zope documentation:

    optional=boolean If true and the variable is null or non-existent, then nothing is inserted.

    So I don't think this should be rendered as NULL when using optional.

    As you pointed out, "None" was saved in some customer project and this is not good neither because None in Python translates to NULL in SQL. So I think the current behavior is good (if we want to follow Zope behavior of course).

Please register or sign in to reply
return 'null'
if type(expr) is not str:
raise
raise ValueError('Missing input variable, <em>%s</em>' % self.__name__)
if v is None and args.get('optional'):
return 'null'
if t == 'int':
try:
if isinstance(v, StringTypes):
if v[-1:] == 'L':
v = v[:-1]
int(v)
else:
v = str(int(v))
except Exception:
if not v and 'optional' in args and args['optional']:
return 'null'
err = 'Invalid integer value for <em>%s</em>' % name
raise ValueError(err)
elif t == 'float':
try:
if isinstance(v, StringTypes):
if v[-1:] == 'L':
v = v[:-1]
float(v)
else:
# ERP5 patch: We use repr that have better precision than str for
# floats (on python2 only)
v = repr(float(v))
except Exception:
if not v and 'optional' in args and args['optional']:
return 'null'
err = 'Invalid floating-point value for <em>%s</em>' % name
raise ValueError(err)
if t=='int':
try:
if type(v) is str:
if v[-1:]=='L':
v=v[:-1]
atoi(v)
return v
return str(int(v))
except Exception:
t = 'integer'
elif t=='float':
try:
if type(v) is str:
if v[-1:]=='L':
v=v[:-1]
atof(v)
return v
# ERP5 patch, we use repr that have better precision than str for
# floats
return repr(float(v))
except Exception:
t = 'floating-point'
elif t.startswith('datetime'):
# For subsecond precision, use 'datetime(N)' MySQL type,
# where N is the number of digits after the decimal point.
n = 0 if t == 'datetime' else int(t[9])
try:
v = (v if isinstance(v, DateTime) else DateTime(v)).toZone('UTC')
return "'%s%s'" % (v.ISO(),
('.%06u' % (v.micros() % 1000000))[:1+n] if n else '')
except Exception:
t = 'datetime'
elif t=='nb' and not v:
t = 'empty string'
else:
v = md.getitem('sql_quote__',0)(
v if isinstance(v, basestring) else str(v))
#if find(v,"\'") >= 0: v=join(split(v,"\'"),"''")
#v="'%s'" % v
return v
elif t.startswith('datetime'):
# For subsecond precision, use 'datetime(N)' MySQL type,
# where N is the number of digits after the decimal point.
n = 0 if t == 'datetime' else int(t[9])
try:
v = (v if isinstance(v, DateTime) else DateTime(v)).toZone('UTC')
return "'%s%s'" % (v.ISO(),
('.%06u' % (v.micros() % 1000000))[:1+n] if n else '')
except Exception:
t = 'datetime'
elif t=='nb' and not v:
t = 'empty string'
if args.get('optional'):
return 'null'
raise ValueError('Invalid %s value for <em>%s</em>: %r'
% (t, self.__name__, v))
else:
if not isinstance(v, (str, StringTypes)):
v = str(v)
if not v and t == 'nb':
if 'optional' in args and args['optional']:
return 'null'
else:
err = 'Invalid empty string value for <em>%s</em>' % name
raise ValueError(err)
v = md.getitem('sql_quote__', 0)(v)
# if v.find("\'") >= 0: v="''".join(v.split("\'"))
# v="'%s'" % v
return v
# Patched by yo. datetime is added.
new_valid_types = 'int', 'float', 'string', 'nb', 'datetime'
new_valid_types += tuple(map('datetime(%s)'.__mod__, xrange(7)))
new_valid_types += tuple(map('datetime(%s)'.__mod__, range(7)))
try:
# BBB
from Shared.DC.ZRDB.sqlvar import valid_type
sqlvar.valid_type = new_valid_types.__contains__
except ImportError:
sqlvar.valid_types = new_valid_types
SQLVar.render = SQLVar_render
SQLVar.__call__ = SQLVar_render
SQLVar.render = render
SQLVar.__call__ = render
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment