Commit fcad0dd8 authored by Jérome Perrin's avatar Jérome Perrin

big_file: py3

parent e4762c12
...@@ -14,7 +14,6 @@ ...@@ -14,7 +14,6 @@
# #
############################################################################## ##############################################################################
from six.moves import cStringIO as StringIO
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5Type.Base import removeIContentishInterface from Products.ERP5Type.Base import removeIContentishInterface
...@@ -23,14 +22,17 @@ from erp5.component.module.BTreeData import BTreeData ...@@ -23,14 +22,17 @@ from erp5.component.module.BTreeData import BTreeData
from ZPublisher.HTTPRequest import FileUpload from ZPublisher.HTTPRequest import FileUpload
from ZPublisher import HTTPRangeSupport from ZPublisher import HTTPRangeSupport
from zope.datetime import rfc1123_date from zope.datetime import rfc1123_date
from mimetools import choose_boundary
from Products.CMFCore.utils import _setCacheHeaders, _ViewEmulator from Products.CMFCore.utils import _setCacheHeaders, _ViewEmulator
from DateTime import DateTime from DateTime import DateTime
import re import re
import io
import six import six
if six.PY3: if six.PY3:
long = int # pylint:disable=redefined-builtin long = int # pylint:disable=redefined-builtin
from email.generator import _make_boundary as choose_boundary
else:
from mimetools import choose_boundary # pylint:disable=import-error
class BigFile(File): class BigFile(File):
""" """
...@@ -43,10 +45,10 @@ class BigFile(File): ...@@ -43,10 +45,10 @@ class BigFile(File):
data property is either data property is either
- BTreeData instance, or - BTreeData instance, or
- str(*), or - bytes(*), or
- None. - None.
(*) str has to be supported because '' is a default value for `data` field (*) bytes has to be supported because b'' is a default value for `data` field
from Data property sheet. from Data property sheet.
Even more - for Even more - for
...@@ -55,7 +57,7 @@ class BigFile(File): ...@@ -55,7 +57,7 @@ class BigFile(File):
b) desire to support automatic migration of File-based documents b) desire to support automatic migration of File-based documents
from document_module to BigFiles from document_module to BigFiles
non-empty str for data also have to be supported. non-empty bytes for data also have to be supported.
XXX(kirr) I'm not sure supporting non-empty str is a good idea (it XXX(kirr) I'm not sure supporting non-empty str is a good idea (it
would be simpler if .data could be either BTreeData or "empty"), would be simpler if .data could be either BTreeData or "empty"),
...@@ -64,6 +66,8 @@ class BigFile(File): ...@@ -64,6 +66,8 @@ class BigFile(File):
We discussed with Romain and settled on "None or str or BTreeData" We discussed with Romain and settled on "None or str or BTreeData"
invariant for now. invariant for now.
notes: for python3 port "str" becomes "bytes", but kirr message was not modified.
""" """
meta_type = 'ERP5 Big File' meta_type = 'ERP5 Big File'
...@@ -115,9 +119,9 @@ class BigFile(File): ...@@ -115,9 +119,9 @@ class BigFile(File):
# of memory. # of memory.
n=1 << 16 n=1 << 16
if isinstance(file, str): if isinstance(file, bytes):
# Big string: cut it into smaller chunks # Big string: cut it into smaller chunks
file = StringIO(file) file = io.BytesIO(file)
if isinstance(file, FileUpload) and not file: if isinstance(file, FileUpload) and not file:
raise ValueError('File not specified') raise ValueError('File not specified')
...@@ -130,9 +134,9 @@ class BigFile(File): ...@@ -130,9 +134,9 @@ class BigFile(File):
if data is None: if data is None:
btree = BTreeData() btree = BTreeData()
elif isinstance(data, str): elif isinstance(data, bytes):
# we'll want to append content to this file - # we'll want to append content to this file -
# - automatically convert str (empty or not) to BTreeData # - automatically convert bytes (empty or not) to BTreeData
btree = BTreeData() btree = BTreeData()
btree.write(data, 0) btree.write(data, 0)
else: else:
...@@ -236,7 +240,7 @@ class BigFile(File): ...@@ -236,7 +240,7 @@ class BigFile(File):
RESPONSE.setStatus(206) # Partial content RESPONSE.setStatus(206) # Partial content
# NOTE data cannot be None here (if it is - ranges are not satisfiable) # NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str): if isinstance(data, bytes):
RESPONSE.write(data[start:end]) RESPONSE.write(data[start:end])
return True return True
for chunk in data.iterate(start, end-start): for chunk in data.iterate(start, end-start):
...@@ -271,22 +275,22 @@ class BigFile(File): ...@@ -271,22 +275,22 @@ class BigFile(File):
RESPONSE.setStatus(206) # Partial content RESPONSE.setStatus(206) # Partial content
for start, end in ranges: for start, end in ranges:
RESPONSE.write('\r\n--%s\r\n' % boundary) RESPONSE.write(('\r\n--%s\r\n' % boundary).encode())
RESPONSE.write('Content-Type: %s\r\n' % RESPONSE.write(('Content-Type: %s\r\n' %
self.content_type) self.content_type).encode())
RESPONSE.write( RESPONSE.write(
'Content-Range: bytes %d-%d/%d\r\n\r\n' % ( ('Content-Range: bytes %d-%d/%d\r\n\r\n' % (
start, end - 1, self.getSize())) start, end - 1, self.getSize())).encode())
# NOTE data cannot be None here (if it is - ranges are not satisfiable) # NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str): if isinstance(data, bytes):
RESPONSE.write(data[start:end]) RESPONSE.write(data[start:end])
else: else:
for chunk in data.iterate(start, end-start): for chunk in data.iterate(start, end-start):
RESPONSE.write(chunk) RESPONSE.write(chunk)
RESPONSE.write('\r\n--%s--\r\n' % boundary) RESPONSE.write(('\r\n--%s--\r\n' % boundary).encode())
return True return True
security.declareProtected(Permissions.View, 'index_html') security.declareProtected(Permissions.View, 'index_html')
...@@ -296,7 +300,7 @@ class BigFile(File): ...@@ -296,7 +300,7 @@ class BigFile(File):
""" """
if self._range_request_handler(REQUEST, RESPONSE): if self._range_request_handler(REQUEST, RESPONSE):
# we served a chunk of content in response to a range request. # we served a chunk of content in response to a range request.
return '' return b''
web_cache_kw = kw.copy() web_cache_kw = kw.copy()
if format is not _MARKER: if format is not _MARKER:
...@@ -327,13 +331,13 @@ class BigFile(File): ...@@ -327,13 +331,13 @@ class BigFile(File):
if data is None: if data is None:
return '' return b''
if isinstance(data, str): if isinstance(data, bytes):
RESPONSE.setBase(None) RESPONSE.setBase(None)
return data return data
for chunk in data.iterate(): for chunk in data.iterate():
RESPONSE.write(chunk) RESPONSE.write(chunk)
return '' return b''
security.declareProtected(Permissions.ModifyPortalContent,'PUT') security.declareProtected(Permissions.ModifyPortalContent,'PUT')
def PUT(self, REQUEST, RESPONSE): def PUT(self, REQUEST, RESPONSE):
......
...@@ -3,6 +3,7 @@ from BTrees.LOBTree import LOBTree ...@@ -3,6 +3,7 @@ from BTrees.LOBTree import LOBTree
from persistent import Persistent from persistent import Persistent
import itertools import itertools
from six.moves import range from six.moves import range
import six
# Maximum memory to allocate for sparse-induced padding. # Maximum memory to allocate for sparse-induced padding.
MAX_PADDING_CHUNK = 2 ** 20 MAX_PADDING_CHUNK = 2 ** 20
...@@ -11,11 +12,13 @@ class PersistentString(Persistent): ...@@ -11,11 +12,13 @@ class PersistentString(Persistent):
def __init__(self, value): def __init__(self, value):
self.value = value self.value = value
def __str__(self): def __bytes__(self):
return self.value return self.value
if six.PY2:
__str__ = __bytes__
# Save place when storing this data in zodb # Save place when storing this data in zodb
__getstate__ = __str__ __getstate__ = __bytes__
__setstate__ = __init__ __setstate__ = __init__
negative_offset_error = ValueError('Negative offset') negative_offset_error = ValueError('Negative offset')
...@@ -110,7 +113,7 @@ class BTreeData(Persistent): ...@@ -110,7 +113,7 @@ class BTreeData(Persistent):
chunk = tree[lower_key] chunk = tree[lower_key]
chunk_end = lower_key + len(chunk.value) chunk_end = lower_key + len(chunk.value)
if chunk_end > offset or ( if chunk_end > offset or (
len(chunk.value) < self._chunk_size and len(chunk.value) < (self._chunk_size or 0) and
chunk_end == offset chunk_end == offset
): ):
key = lower_key key = lower_key
...@@ -137,7 +140,7 @@ class BTreeData(Persistent): ...@@ -137,7 +140,7 @@ class BTreeData(Persistent):
try: try:
chunk = tree[key] chunk = tree[key]
except KeyError: except KeyError:
tree[key] = chunk = PersistentString('') tree[key] = chunk = PersistentString(b'')
entry_size = len(chunk.value) entry_size = len(chunk.value)
if entry_size < to_write_len: if entry_size < to_write_len:
to_write_len = min(to_write_len, max_to_write_len) to_write_len = min(to_write_len, max_to_write_len)
...@@ -158,9 +161,9 @@ class BTreeData(Persistent): ...@@ -158,9 +161,9 @@ class BTreeData(Persistent):
size (int) size (int)
Number of bytes to read. Number of bytes to read.
Returns string of read data. Returns bytes of read data.
""" """
return ''.join(self.iterate(offset, size)) return b''.join(self.iterate(offset, size))
def iterate(self, offset=0, size=None): def iterate(self, offset=0, size=None):
""" """
...@@ -243,7 +246,7 @@ class BTreeData(Persistent): ...@@ -243,7 +246,7 @@ class BTreeData(Persistent):
except ValueError: except ValueError:
break break
del tree[key] del tree[key]
self.write('', offset) self.write(b'', offset)
# XXX: Various batch_size values need to be benchmarked, and a saner # XXX: Various batch_size values need to be benchmarked, and a saner
# default is likely to be applied. # default is likely to be applied.
...@@ -314,12 +317,11 @@ class BTreeData(Persistent): ...@@ -314,12 +317,11 @@ class BTreeData(Persistent):
tree[key] = next_chunk tree[key] = next_chunk
if __name__ == '__main__': if __name__ == '__main__':
def check(tree, length, read_offset, read_length, data_, keys=None): def check(tree, length, read_offset, read_length, data_, keys=None):
print(list(tree._tree.items())) print(list(tree._tree.items()))
tree_length = len(tree) tree_length = len(tree)
tree_data = tree.read(read_offset, read_length) tree_data = tree.read(read_offset, read_length)
tree_iterator_data = ''.join(tree.iterate(read_offset, read_length)) tree_iterator_data = b''.join(tree.iterate(read_offset, read_length))
assert tree_length == length, tree_length assert tree_length == length, tree_length
assert tree_data == data_, repr(tree_data) assert tree_data == data_, repr(tree_data)
assert tree_iterator_data == data_, repr(tree_iterator_data) assert tree_iterator_data == data_, repr(tree_iterator_data)
......
...@@ -24,8 +24,7 @@ ...@@ -24,8 +24,7 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from six.moves import cStringIO as StringIO import io
from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.HTTPRequest import HTTPRequest
from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.HTTPResponse import HTTPResponse
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
...@@ -35,15 +34,15 @@ from erp5.component.module.BTreeData import BTreeData ...@@ -35,15 +34,15 @@ from erp5.component.module.BTreeData import BTreeData
# like Testing.makerequest, but # like Testing.makerequest, but
# #
# 1. always redirects stdout to stringio, # 1. always redirects stdout to BytesIO,
# 2. stdin content can be specified and is processed, # 2. stdin content can be specified and is processed,
# 3. returns actual request object (not wrapped portal). # 3. returns actual request object (not wrapped portal).
# #
# see also: Products.CMFCore.tests.test_CookieCrumbler.makerequest() # see also: Products.CMFCore.tests.test_CookieCrumbler.makerequest()
# TODO makerequest() is generic enough and deserves moving to testing utils # TODO makerequest() is generic enough and deserves moving to testing utils
def makerequest(environ=None, stdin=''): def makerequest(environ=None, stdin=b''):
stdout = StringIO() stdout = io.BytesIO()
stdin = StringIO(stdin) stdin = io.BytesIO(stdin)
if environ is None: if environ is None:
environ = {} environ = {}
...@@ -65,7 +64,7 @@ def makerequest(environ=None, stdin=''): ...@@ -65,7 +64,7 @@ def makerequest(environ=None, stdin=''):
# generate makerequest-like function for http method # generate makerequest-like function for http method
def request_function(method_name): def request_function(method_name):
method_name = method_name.upper() method_name = method_name.upper()
def method_func(environ=None, stdin=''): def method_func(environ=None, stdin=b''):
if environ is None: if environ is None:
environ = {} environ = {}
environ.setdefault('REQUEST_METHOD', method_name) environ.setdefault('REQUEST_METHOD', method_name)
...@@ -112,7 +111,7 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -112,7 +111,7 @@ class TestBigFile(ERP5TypeTestCase):
ret = method (request, request.RESPONSE, **kw) ret = method (request, request.RESPONSE, **kw)
# like in ZPublisher - returned RESPONSE means empty # like in ZPublisher - returned RESPONSE means empty
if ret is request.RESPONSE: if ret is request.RESPONSE:
ret = '' ret = b''
self.assertEqual(ret, result) self.assertEqual(ret, result)
self.assertEqual(status, request.RESPONSE.getStatus()) self.assertEqual(status, request.RESPONSE.getStatus())
for h,v in header_dict.items(): for h,v in header_dict.items():
...@@ -120,9 +119,9 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -120,9 +119,9 @@ class TestBigFile(ERP5TypeTestCase):
self.assertEqual(v, rv, '%s: %r != %r' % (h, v, rv)) self.assertEqual(v, rv, '%s: %r != %r' % (h, v, rv))
# force response flush to its stdout # force response flush to its stdout
request.RESPONSE.write('') request.RESPONSE.write(b'')
# body and headers are delimited by empty line (RFC 2616, 4.1) # body and headers are delimited by empty line (RFC 2616, 4.1)
response_body = request.RESPONSE.stdout.getvalue().split('\r\n\r\n', 1)[1] response_body = request.RESPONSE.stdout.getvalue().split(b'\r\n\r\n', 1)[1]
self.assertEqual(body, response_body) self.assertEqual(body, response_body)
...@@ -135,90 +134,90 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -135,90 +134,90 @@ class TestBigFile(ERP5TypeTestCase):
# after creation file is empty and get(0-0) returns 416 # after creation file is empty and get(0-0) returns 416
self.assertEqual(f.getSize(), 0) self.assertEqual(f.getSize(), 0)
self.assertEqual(f.getData(), '') self.assertEqual(f.getData(), b'')
# result body status headers # result body status headers
check(get(), {'format': 'raw'}, '', '', 200, {'Content-Length': '0'}) check(get(), {'format': 'raw'}, b'', b'', 200, {'Content-Length': '0'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok? check(put({'Content-Range': 'bytes */*'}),{}, b'', b'',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok?
check(get({ 'Range': 'bytes=0-0'}),{}, '', '', 416, {'Content-Length': '0', 'Content-Range': 'bytes */0'}) check(get({ 'Range': 'bytes=0-0'}),{}, b'', b'', 416, {'Content-Length': '0', 'Content-Range': 'bytes */0'})
# append empty chunk - the same # append empty chunk - the same
f._appendData('') f._appendData(b'')
self.assertEqual(f.getSize(), 0) self.assertEqual(f.getSize(), 0)
self.assertEqual(f.getData(), '') self.assertEqual(f.getData(), b'')
check(get(), {'format': 'raw'}, '', '', 200, {'Content-Length': '0'}) check(get(), {'format': 'raw'}, b'', b'', 200, {'Content-Length': '0'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok? check(put({'Content-Range': 'bytes */*'}),{}, b'', b'',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok?
check(get({ 'Range': 'bytes=0-0'}),{}, '', '', 416, { 'Content-Range': 'bytes */0'}) check(get({ 'Range': 'bytes=0-0'}),{}, b'', b'', 416, { 'Content-Range': 'bytes */0'})
# append 1 byte - file should grow up and get(0-0) returns 206 # append 1 byte - file should grow up and get(0-0) returns 206
f._appendData('x') f._appendData(b'x')
self.assertEqual(f.getSize(), 1) self.assertEqual(f.getSize(), 1)
self.assertEqual(f.getData(), 'x') self.assertEqual(f.getData(), b'x')
check(get(), {'format': 'raw'}, '', 'x', 200, {'Content-Length': '1'}) check(get(), {'format': 'raw'}, b'', b'x', 200, {'Content-Length': '1'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-0'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-0'})
check(get({ 'Range': 'bytes=0-0'}),{}, '', 'x', 206, {'Content-Length': '1', 'Content-Range': 'bytes 0-0/1'}) check(get({ 'Range': 'bytes=0-0'}),{}, b'', b'x', 206, {'Content-Length': '1', 'Content-Range': 'bytes 0-0/1'})
# append another 2 bytes and try to get whole file and partial contents # append another 2 bytes and try to get whole file and partial contents
f._appendData('yz') f._appendData(b'yz')
self.assertEqual(f.getSize(), 3) self.assertEqual(f.getSize(), 3)
self.assertEqual(f.getData(), 'xyz') self.assertEqual(f.getData(), b'xyz')
check(get(), {'format': 'raw'}, '', 'xyz', 200, {'Content-Length': '3'}) check(get(), {'format': 'raw'}, b'', b'xyz', 200, {'Content-Length': '3'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-2'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-2'})
check(get({ 'Range': 'bytes=0-0'}),{}, '', 'x' , 206, {'Content-Length': '1', 'Content-Range': 'bytes 0-0/3'}) check(get({ 'Range': 'bytes=0-0'}),{}, b'', b'x' , 206, {'Content-Length': '1', 'Content-Range': 'bytes 0-0/3'})
check(get({ 'Range': 'bytes=1-1'}),{}, '', 'y' , 206, {'Content-Length': '1', 'Content-Range': 'bytes 1-1/3'}) check(get({ 'Range': 'bytes=1-1'}),{}, b'', b'y' , 206, {'Content-Length': '1', 'Content-Range': 'bytes 1-1/3'})
check(get({ 'Range': 'bytes=2-2'}),{}, '', 'z', 206, {'Content-Length': '1', 'Content-Range': 'bytes 2-2/3'}) check(get({ 'Range': 'bytes=2-2'}),{}, b'', b'z', 206, {'Content-Length': '1', 'Content-Range': 'bytes 2-2/3'})
check(get({ 'Range': 'bytes=0-1'}),{}, '', 'xy' , 206, {'Content-Length': '2', 'Content-Range': 'bytes 0-1/3'}) check(get({ 'Range': 'bytes=0-1'}),{}, b'', b'xy' , 206, {'Content-Length': '2', 'Content-Range': 'bytes 0-1/3'})
check(get({ 'Range': 'bytes=1-2'}),{}, '', 'yz', 206, {'Content-Length': '2', 'Content-Range': 'bytes 1-2/3'}) check(get({ 'Range': 'bytes=1-2'}),{}, b'', b'yz', 206, {'Content-Length': '2', 'Content-Range': 'bytes 1-2/3'})
check(get({ 'Range': 'bytes=0-2'}),{}, '', 'xyz', 206, {'Content-Length': '3', 'Content-Range': 'bytes 0-2/3'}) check(get({ 'Range': 'bytes=0-2'}),{}, b'', b'xyz', 206, {'Content-Length': '3', 'Content-Range': 'bytes 0-2/3'})
# append via PUT with range # append via PUT with range
check(put({'Content-Range': 'bytes 3-4/5', 'Content-Length': '2'}, '01'),{}, '', '', 204, {}) check(put({'Content-Range': 'bytes 3-4/5', 'Content-Length': '2'}, b'01'),{}, b'', b'', 204, {})
self.assertEqual(f.getSize(), 5) self.assertEqual(f.getSize(), 5)
self.assertEqual(f.getData(), 'xyz01') self.assertEqual(f.getData(), b'xyz01')
check(get(), {'format': 'raw'}, '', 'xyz01',200, {'Content-Length': '5'}) check(get(), {'format': 'raw'}, b'', b'xyz01',200, {'Content-Length': '5'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-4'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-4'})
check(get({ 'Range': 'bytes=0-4'}),{}, '', 'xyz01',206, {'Content-Length': '5', 'Content-Range': 'bytes 0-4/5'}) check(get({ 'Range': 'bytes=0-4'}),{}, b'', b'xyz01',206, {'Content-Length': '5', 'Content-Range': 'bytes 0-4/5'})
check(get({ 'Range': 'bytes=1-3'}),{}, '', 'yz0' ,206, {'Content-Length': '3', 'Content-Range': 'bytes 1-3/5'}) check(get({ 'Range': 'bytes=1-3'}),{}, b'', b'yz0' ,206, {'Content-Length': '3', 'Content-Range': 'bytes 1-3/5'})
check(get({ 'Range': 'bytes=1-2'}),{}, '', 'yz' ,206, {'Content-Length': '2', 'Content-Range': 'bytes 1-2/5'}) check(get({ 'Range': 'bytes=1-2'}),{}, b'', b'yz' ,206, {'Content-Length': '2', 'Content-Range': 'bytes 1-2/5'})
check(get({ 'Range': 'bytes=2-2'}),{}, '', 'z' ,206, {'Content-Length': '1', 'Content-Range': 'bytes 2-2/5'}) check(get({ 'Range': 'bytes=2-2'}),{}, b'', b'z' ,206, {'Content-Length': '1', 'Content-Range': 'bytes 2-2/5'})
# replace whole content via PUT without range # replace whole content via PUT without range
# (and we won't exercise GET with range routinely further) # (and we won't exercise GET with range routinely further)
check(put({'Content-Length': '3'}, 'abc'),{}, '', '', 204, {}) check(put({'Content-Length': '3'}, b'abc'),{}, b'', b'', 204, {})
self.assertEqual(f.getSize(), 3) self.assertEqual(f.getSize(), 3)
self.assertEqual(f.getData(), 'abc') self.assertEqual(f.getData(), b'abc')
check(get(), {'format': 'raw'}, '', 'abc', 200, {'Content-Length': '3'}) check(get(), {'format': 'raw'}, b'', b'abc', 200, {'Content-Length': '3'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-2'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-2'})
# append via PUT with range (again) # append via PUT with range (again)
check(put({'Content-Range': 'bytes 3-7/8', 'Content-Length': '5'}, 'defgh'),{}, '', '', 204, {}) check(put({'Content-Range': 'bytes 3-7/8', 'Content-Length': '5'}, b'defgh'),{}, b'', b'', 204, {})
self.assertEqual(f.getSize(), 8) self.assertEqual(f.getSize(), 8)
self.assertEqual(f.getData(), 'abcdefgh') self.assertEqual(f.getData(), b'abcdefgh')
check(get(), {'format': 'raw'}, '', 'abcdefgh', 200, {'Content-Length': '8'}) check(get(), {'format': 'raw'}, b'', b'abcdefgh', 200, {'Content-Length': '8'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-7'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-7'})
# append via ._appendData() (again) # append via ._appendData() (again)
f._appendData('ij') f._appendData(b'ij')
self.assertEqual(f.getSize(), 10) self.assertEqual(f.getSize(), 10)
self.assertEqual(f.getData(), 'abcdefghij') self.assertEqual(f.getData(), b'abcdefghij')
check(get(), {'format': 'raw'}, '', 'abcdefghij', 200, {'Content-Length': '10'}) check(get(), {'format': 'raw'}, b'', b'abcdefghij', 200, {'Content-Length': '10'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-9'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-9'})
# make sure PUT with incorrect/non-append range is rejected # make sure PUT with incorrect/non-append range is rejected
check(put({'Content-Range': 'bytes 10-10/10', 'Content-Length': '1'}, 'k'),{}, '', '', 400, {'X-Explanation': 'Total size unexpected'}) check(put({'Content-Range': 'bytes 10-10/10', 'Content-Length': '1'}, b'k'),{}, b'', b'', 400, {'X-Explanation': 'Total size unexpected'})
self.assertEqual(f.getData(), 'abcdefghij') self.assertEqual(f.getData(), b'abcdefghij')
check(put({'Content-Range': 'bytes 10-10/11', 'Content-Length': '2'}, 'k'),{}, '', '', 400, {'X-Explanation': 'Content length unexpected'}) check(put({'Content-Range': 'bytes 10-10/11', 'Content-Length': '2'}, b'k'),{}, b'', b'', 400, {'X-Explanation': 'Content length unexpected'})
self.assertEqual(f.getData(), 'abcdefghij') self.assertEqual(f.getData(), b'abcdefghij')
check(put({'Content-Range': 'bytes 8-8/10', 'Content-Length': '1'}, '?'),{}, '', '', 400, {'X-Explanation': 'Can only append data'}) check(put({'Content-Range': 'bytes 8-8/10', 'Content-Length': '1'}, b'?'),{}, b'', b'', 400, {'X-Explanation': 'Can only append data'})
check(put({'Content-Range': 'bytes 9-9/10', 'Content-Length': '1'}, '?'),{}, '', '', 400, {'X-Explanation': 'Can only append data'}) check(put({'Content-Range': 'bytes 9-9/10', 'Content-Length': '1'}, b'?'),{}, b'', b'', 400, {'X-Explanation': 'Can only append data'})
check(put({'Content-Range': 'bytes 9-10/11', 'Content-Length': '2'},'??'),{}, '', '', 400, {'X-Explanation': 'Can only append data'}) check(put({'Content-Range': 'bytes 9-10/11', 'Content-Length': '2'},b'??'),{}, b'', b'', 400, {'X-Explanation': 'Can only append data'})
check(put({'Content-Range': 'bytes 11-11/12', 'Content-Length': '1'}, '?'),{}, '', '', 400, {'X-Explanation': 'Can only append data'}) check(put({'Content-Range': 'bytes 11-11/12', 'Content-Length': '1'}, b'?'),{}, b'', b'', 400, {'X-Explanation': 'Can only append data'})
# TODO test 'If-Range' with GET # TODO test 'If-Range' with GET
# TODO test multiple ranges in 'Range' with GET # TODO test multiple ranges in 'Range' with GET
...@@ -229,11 +228,11 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -229,11 +228,11 @@ class TestBigFile(ERP5TypeTestCase):
# (called from under testBigFile_02_DataVarious driver) # (called from under testBigFile_02_DataVarious driver)
def _testBigFile_02_DataVarious(self): def _testBigFile_02_DataVarious(self):
# BigFile's .data can be: # BigFile's .data can be:
# str - because data comes from Data property sheet and default value is '' # bytes - because data comes from Data property sheet and default value is b''
# None - because it can be changed # None - because it can be changed
# BTreeData - because it is scalable way to work with large content # BTreeData - because it is scalable way to work with large content
# #
# str can be possibly non-empty because we could want to transparently # bytes can be possibly non-empty because we could want to transparently
# migrate plain File documents to BigFiles. # migrate plain File documents to BigFiles.
# #
# make sure BigFile correctly works in all those situations. # make sure BigFile correctly works in all those situations.
...@@ -242,47 +241,47 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -242,47 +241,47 @@ class TestBigFile(ERP5TypeTestCase):
f = big_file_module.newContent(portal_type='Big File') f = big_file_module.newContent(portal_type='Big File')
check = lambda *args: self.checkRequest(f, *args) check = lambda *args: self.checkRequest(f, *args)
# after creation .data is '' (as per default from Data property sheet) # after creation .data is b'' (as per default from Data property sheet)
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, str) self.assertIsInstance(_, bytes)
self.assertEqual(_, '') self.assertEqual(_, b'')
# make sure we can get empty content through all ways # make sure we can get empty content through all ways
# (already covered in testBigFile_01_Basic, but still) # (already covered in testBigFile_01_Basic, but still)
self.assertEqual(f.getSize(), 0) self.assertEqual(f.getSize(), 0)
self.assertEqual(f.getData(), '') self.assertEqual(f.getData(), b'')
check(get(), {'format': 'raw'}, '', '', 200, {'Content-Length': '0'}) check(get(), {'format': 'raw'}, b'', b'', 200, {'Content-Length': '0'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok? check(put({'Content-Range': 'bytes */*'}),{}, b'', b'',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok?
check(get({ 'Range': 'bytes=0-0'}),{}, '', '', 416, {'Content-Length': '0', 'Content-Range': 'bytes */0'}) check(get({ 'Range': 'bytes=0-0'}),{}, b'', b'', 416, {'Content-Length': '0', 'Content-Range': 'bytes */0'})
# set .data to non-empty str and make sure we can get content through all ways # set .data to non-empty str and make sure we can get content through all ways
f._baseSetData('abc') f._baseSetData(b'abc')
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, str) self.assertIsInstance(_, bytes)
self.assertEqual(_, 'abc') self.assertEqual(_, b'abc')
self.assertEqual(f.getSize(), 3) self.assertEqual(f.getSize(), 3)
self.assertEqual(f.getData(), 'abc') self.assertEqual(f.getData(), b'abc')
check(get(), {'format': 'raw'}, 'abc', '', 200, {'Content-Length': '3'}) check(get(), {'format': 'raw'}, b'abc', b'', 200, {'Content-Length': '3'})
check(put({'Content-Range': 'bytes */*'}),{}, '' , '', R308, { 'Range': 'bytes 0-2'}) check(put({'Content-Range': 'bytes */*'}),{}, b'' , b'', R308, { 'Range': 'bytes 0-2'})
check(get({ 'Range': 'bytes=0-2'}),{}, '' , 'abc', 206, {'Content-Length': '3', 'Content-Range': 'bytes 0-2/3'}) check(get({ 'Range': 'bytes=0-2'}),{}, b'' , b'abc', 206, {'Content-Length': '3', 'Content-Range': 'bytes 0-2/3'})
# and .data should remain str after access (though later this could be # and .data should remain str after access (though later this could be
# changed to transparently migrate to BTreeData) # changed to transparently migrate to BTreeData)
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, str) self.assertIsInstance(_, bytes)
self.assertEqual(_, 'abc') self.assertEqual(_, b'abc')
# on append .data should migrate to BTreeData # on append .data should migrate to BTreeData
f._appendData('d') f._appendData(b'd')
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, BTreeData) self.assertIsInstance(_, BTreeData)
self.assertEqual(f.getSize(), 4) self.assertEqual(f.getSize(), 4)
self.assertEqual(f.getData(), 'abcd') self.assertEqual(f.getData(), b'abcd')
check(get(), {'format': 'raw'}, '', 'abcd', 200, {'Content-Length': '4'}) check(get(), {'format': 'raw'}, b'', b'abcd', 200, {'Content-Length': '4'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-3'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-3'})
check(get({ 'Range': 'bytes=0-3'}),{}, '', 'abcd', 206, {'Content-Length': '4', 'Content-Range': 'bytes 0-3/4'}) check(get({ 'Range': 'bytes=0-3'}),{}, b'', b'abcd', 206, {'Content-Length': '4', 'Content-Range': 'bytes 0-3/4'})
...@@ -292,39 +291,39 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -292,39 +291,39 @@ class TestBigFile(ERP5TypeTestCase):
# #
# see ERP5.Document.File._setSize() for details. # see ERP5.Document.File._setSize() for details.
f._setData(None) f._setData(None)
# NOTE still '' because it is default value specified in Data property # NOTE still b'' because it is default value specified in Data property
# sheet for .data field # sheet for .data field
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, str) self.assertIsInstance(_, bytes)
self.assertEqual(_, '') self.assertEqual(_, b'')
# but we can change property sheet default on the fly # but we can change property sheet default on the fly
# XXX ( only for this particular getter _baseGetData - # XXX ( only for this particular getter _baseGetData -
# - because property type information is not stored in one place, # - because property type information is not stored in one place,
# but is copied on getter/setter initialization - see Getter/Setter # but is copied on getter/setter initialization - see Getter/Setter
# in ERP5Type.Accessor.Base ) # in ERP5Type.Accessor.Base )
# NOTE this change is automatically reverted back in calling helper # NOTE this change is automatically reverted back in calling helper
self.assertIsInstance(f._baseGetData._default, str) self.assertIsInstance(f._baseGetData._default, bytes)
self.assertEqual(f._baseGetData._default, '') self.assertEqual(f._baseGetData._default, b'')
f._baseGetData.__func__._default = None # NOTE not possible to do on just f._baseGetData f._baseGetData.__func__._default = None # NOTE not possible to do on just f._baseGetData
self.assertIs(f._baseGetData._default, None) self.assertIs(f._baseGetData._default, None)
self.assertIs(f._baseGetData(), None) # <- oops self.assertIs(f._baseGetData(), None) # <- oops
self.assertEqual(f.getSize(), 0) self.assertEqual(f.getSize(), 0)
self.assertIs (f.getData(), None) self.assertIs (f.getData(), None)
check(get(), {'format': 'raw'}, '', '', 200, {'Content-Length': '0'}) check(get(), {'format': 'raw'}, b'', b'', 200, {'Content-Length': '0'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok? check(put({'Content-Range': 'bytes */*'}),{}, b'', b'',R308, { 'Range': 'bytes 0--1'}) # XXX 0--1 ok?
check(get({ 'Range': 'bytes=0-0'}),{}, '', '', 416, {'Content-Length': '0', 'Content-Range': 'bytes */0'}) check(get({ 'Range': 'bytes=0-0'}),{}, b'', b'', 416, {'Content-Length': '0', 'Content-Range': 'bytes */0'})
# on append .data should become BTreeData # on append .data should become BTreeData
f._appendData('x') f._appendData(b'x')
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, BTreeData) self.assertIsInstance(_, BTreeData)
self.assertEqual(f.getSize(), 1) self.assertEqual(f.getSize(), 1)
self.assertEqual(f.getData(), 'x') self.assertEqual(f.getData(), b'x')
check(get(), {'format': 'raw'}, '', 'x', 200, {'Content-Length': '1'}) check(get(), {'format': 'raw'}, b'', b'x', 200, {'Content-Length': '1'})
check(put({'Content-Range': 'bytes */*'}),{}, '', '', R308, { 'Range': 'bytes 0-0'}) check(put({'Content-Range': 'bytes */*'}),{}, b'', b'', R308, { 'Range': 'bytes 0-0'})
check(get({ 'Range': 'bytes=0-3'}),{}, '', 'x', 206, {'Content-Length': '1', 'Content-Range': 'bytes 0-0/1'}) check(get({ 'Range': 'bytes=0-3'}),{}, b'', b'x', 206, {'Content-Length': '1', 'Content-Range': 'bytes 0-0/1'})
...@@ -333,26 +332,26 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -333,26 +332,26 @@ class TestBigFile(ERP5TypeTestCase):
big_file_module = self.getPortal().big_file_module big_file_module = self.getPortal().big_file_module
f = big_file_module.newContent(portal_type='Big File') f = big_file_module.newContent(portal_type='Big File')
# Data property sheet specifies .data default to '' # Data property sheet specifies .data default to b''
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, str) self.assertIsInstance(_, bytes)
self.assertEqual(_, '') self.assertEqual(_, b'')
# NOTE obtaining getter is not possible via BigFile._baseGetData # NOTE obtaining getter is not possible via BigFile._baseGetData
g = f._baseGetData.__func__ g = f._baseGetData.__func__
self.assertIsInstance(g._default, str) self.assertIsInstance(g._default, bytes)
self.assertEqual(g._default, '') self.assertEqual(g._default, b'')
try: try:
self._testBigFile_02_DataVarious() self._testBigFile_02_DataVarious()
# restore ._baseGetData._default and make sure restoration really worked # restore ._baseGetData._default and make sure restoration really worked
finally: finally:
g._default = '' g._default = b''
f._baseSetData(None) # so that we are sure getter returns class defaults f._baseSetData(None) # so that we are sure getter returns class defaults
_ = f._baseGetData() _ = f._baseGetData()
self.assertIsInstance(_, str) self.assertIsInstance(_, bytes)
self.assertEqual(_, '') self.assertEqual(_, b'')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment