1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# 2012 Nexedi SA and Contributors. All Rights Reserved.
# Romain Courteaud <romain@nexedi.com>
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from cStringIO import StringIO
from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5Type.Base import removeIContentishInterface
from Products.ERP5.Document.File import File, _MARKER
from Products.ERP5Type.BTreeData import BTreeData
from ZPublisher.HTTPRequest import FileUpload
from ZPublisher import HTTPRangeSupport
from webdav.common import rfc1123_date
from mimetools import choose_boundary
from Products.CMFCore.utils import getToolByName, _setCacheHeaders,\
_ViewEmulator
import re
class BigFile(File):
"""
Support storing huge file.
No convertion is allowed for now.
NOTE BigFile maintains the following invariant:
data property is either
- BTreeData instance, or
- str(*), or
- None.
(*) str has to be supported because '' is a default value for `data` field
from Data property sheet.
Even more - for
a) compatibility reasons, and
b) desire to support automatic migration of File-based documents
from document_module to BigFiles
non-empty str for data also have to be supported.
XXX(kirr) I'm not sure supporting non-empty str is a good idea (it
would be simpler if .data could be either BTreeData or "empty"),
but neither I'm experienced enough in erp5 nor know what are
appropriate compatibility requirements.
We discussed with Romain and settled on "None or str or BTreeData"
invariant for now.
"""
meta_type = 'ERP5 Big File'
portal_type = 'Big File'
# Declarative security
security = ClassSecurityInfo()
security.declareObjectProtected(Permissions.AccessContentsInformation)
# Default Properties
property_sheets = ( PropertySheet.Base
, PropertySheet.XMLObject
, PropertySheet.CategoryCore
, PropertySheet.DublinCore
, PropertySheet.Version
, PropertySheet.Reference
, PropertySheet.Document
, PropertySheet.Data
, PropertySheet.ExternalDocument
, PropertySheet.Url
, PropertySheet.Periodicity
)
# OFS.File has an overloaded __str__ that returns the file content
__str__ = object.__str__
security.declareProtected(Permissions.AccessContentsInformation,
'getData')
def getData(self):
"""Read the full btree
"""
btree = self._baseGetData()
if isinstance(btree, BTreeData):
return btree.read(0, len(btree))
else:
return btree
security.declareProtected(Permissions.ModifyPortalContent, 'updateContentMd5')
def updateContentMd5(self):
"""Update md5 checksum from the original file
"""
self._setContentMd5(None)
def _read_data(self, file, data=None, serialize=True):
# We might need to make this value configurable. It is important to
# consider the max quantity of object used in the cache. With a default
# cache of 5000 objects, and with n = 64KB, this makes using about 330 MB
# of memory.
n=1 << 16
if isinstance(file, str):
# Big string: cut it into smaller chunks
file = StringIO(file)
if isinstance(file, FileUpload) and not file:
raise ValueError, 'File not specified'
seek=file.seek
read=file.read
seek(0,2)
end=file.tell()
if data is None:
btree = BTreeData()
elif isinstance(data, str):
# we'll want to append content to this file -
# - automatically convert str (empty or not) to BTreeData
btree = BTreeData()
btree.write(data, 0)
else:
btree = data
seek(0)
pos = file.tell()
offset = len(btree)
while pos < end:
next = pos + n
if next > end:
next = end
btree.write(read(next-pos), offset+pos)
pos = file.tell()
if serialize:
self.serialize()
return btree, len(btree)
def _data_mtime(self):
"""get .data mtime if present and fallback to self._p_mtime"""
# there is no data._p_mtime when data is None or str.
# so try and fallback to self._p_mtime
data = self._baseGetData()
mtime = getattr(data, '_p_mtime', self._p_mtime)
return mtime
def _range_request_handler(self, REQUEST, RESPONSE):
# HTTP Range header handling: return True if we've served a range
# chunk out of our data.
range = REQUEST.get_header('Range', None)
request_range = REQUEST.get_header('Request-Range', None)
if request_range is not None:
# Netscape 2 through 4 and MSIE 3 implement a draft version
# Later on, we need to serve a different mime-type as well.
range = request_range
if_range = REQUEST.get_header('If-Range', None)
if range is not None:
ranges = HTTPRangeSupport.parseRange(range)
data = self._baseGetData()
if if_range is not None:
# Only send ranges if the data isn't modified, otherwise send
# the whole object. Support both ETags and Last-Modified dates!
if len(if_range) > 1 and if_range[:2] == 'ts':
# ETag:
if if_range != self.http__etag():
# Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200
# response.
ranges = None
else:
# Date
date = if_range.split( ';')[0]
try: mod_since=long(DateTime(date).timeTime())
except: mod_since=None
if mod_since is not None:
last_mod = self._data_mtime()
if last_mod is None:
last_mod = 0
last_mod = long(last_mod)
if last_mod > mod_since:
# Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200
# response.
ranges = None
if ranges:
# Search for satisfiable ranges.
satisfiable = 0
for start, end in ranges:
if start < self.getSize():
satisfiable = 1
break
if not satisfiable:
RESPONSE.setHeader('Content-Range',
'bytes */%d' % self.getSize())
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime()))
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', self.getSize())
RESPONSE.setStatus(416)
return True
ranges = HTTPRangeSupport.expandRanges(ranges, self.getSize())
if len(ranges) == 1:
# Easy case, set extra header and return partial set.
start, end = ranges[0]
size = end - start
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime()))
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Content-Range',
'bytes %d-%d/%d' % (start, end - 1, self.getSize()))
RESPONSE.setStatus(206) # Partial content
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str):
RESPONSE.write(data[start:end])
return True
for chunk in data.iterate(start, end-start):
RESPONSE.write(chunk)
return True
else:
boundary = choose_boundary()
# Calculate the content length
size = (8 + len(boundary) + # End marker length
len(ranges) * ( # Constant lenght per set
49 + len(boundary) + len(self.content_type) +
len('%d' % self.getSize())))
for start, end in ranges:
# Variable length per set
size = (size + len('%d%d' % (start, end - 1)) +
end - start)
data = self._baseGetData()
# Some clients implement an earlier draft of the spec, they
# will only accept x-byteranges.
draftprefix = (request_range is not None) and 'x-' or ''
RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime()))
RESPONSE.setHeader('Content-Type',
'multipart/%sbyteranges; boundary=%s' % (
draftprefix, boundary))
RESPONSE.setStatus(206) # Partial content
for start, end in ranges:
RESPONSE.write('\r\n--%s\r\n' % boundary)
RESPONSE.write('Content-Type: %s\r\n' %
self.content_type)
RESPONSE.write(
'Content-Range: bytes %d-%d/%d\r\n\r\n' % (
start, end - 1, self.getSize()))
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str):
RESPONSE.write(data[start:end])
else:
for chunk in data.iterate(start, end-start):
RESPONSE.write(chunk)
RESPONSE.write('\r\n--%s--\r\n' % boundary)
return True
security.declareProtected(Permissions.View, 'index_html')
def index_html(self, REQUEST, RESPONSE, format=_MARKER, inline=_MARKER, **kw):
"""
Support streaming
"""
if self._range_request_handler(REQUEST, RESPONSE):
# we served a chunk of content in response to a range request.
return ''
web_cache_kw = kw.copy()
if format is not _MARKER:
web_cache_kw['format'] = format
_setCacheHeaders(_ViewEmulator().__of__(self), web_cache_kw)
if format is _MARKER and not kw:
# conversion parameters is mandatory to download the converted content.
# By default allways return view action.
# for all WevDAV access return raw content.
return self.view()
if format is _MARKER:
format = None
data = self._baseGetData()
mime = self.getContentType()
RESPONSE.setHeader('Content-Length', data is not None and len(data) or 0)
RESPONSE.setHeader('Content-Type', mime)
if inline is _MARKER:
# by default, use inline for text and image formats
inline = False
if not inline:
# need to return it as attachment
filename = self.getStandardFilename(format=format)
RESPONSE.setHeader('Cache-Control', 'Private') # workaround for Internet Explorer's bug
RESPONSE.setHeader('Accept-Ranges', 'bytes')
if data is None:
return ''
if isinstance(data, str):
RESPONSE.setBase(None)
return data
for chunk in data.iterate():
RESPONSE.write(chunk)
return ''
security.declareProtected(Permissions.ModifyPortalContent,'PUT')
def PUT(self, REQUEST, RESPONSE):
"""Handle HTTP PUT requests"""
self.dav__init(REQUEST, RESPONSE)
self.dav__simpleifhandler(REQUEST, RESPONSE, refresh=1)
type=REQUEST.get_header('content-type', None)
file=REQUEST['BODYFILE']
content_range = REQUEST.get_header('Content-Range', None)
if content_range is None:
# truncate the file
self._baseSetData(None)
else:
current_size = int(self.getSize())
query_range = re.compile('bytes \*/\*')
append_range = re.compile('bytes (?P<first_byte>[0-9]+)-' \
'(?P<last_byte>[0-9]+)/' \
'(?P<total_content_length>[0-9]+)')
if query_range.match(content_range):
data = self._baseGetData()
RESPONSE.setHeader('X-Explanation', 'Resume incomplete')
RESPONSE.setHeader('Range', 'bytes 0-%s' % (current_size-1))
RESPONSE.setStatus(308)
return RESPONSE
if append_range.match(content_range):
result_dict = append_range.search(content_range).groupdict()
first_byte = int(result_dict['first_byte'])
last_byte = int(result_dict['last_byte'])
total_content_length = int(result_dict['total_content_length'])
content_length= int(REQUEST.get_header('Content-Length', '0'))
if (first_byte != current_size):
RESPONSE.setHeader('X-Explanation', 'Can only append data')
RESPONSE.setStatus(400)
return RESPONSE
elif (last_byte+1 != total_content_length):
RESPONSE.setHeader('X-Explanation', 'Total size unexpected')
RESPONSE.setStatus(400)
return RESPONSE
elif (last_byte+1-first_byte != content_length):
RESPONSE.setHeader('X-Explanation', 'Content length unexpected')
RESPONSE.setStatus(400)
return RESPONSE
else:
RESPONSE.setHeader('X-Explanation', 'Can not parse range')
RESPONSE.setStatus(400) # Partial content
return RESPONSE
self._appendData(file, content_type=type)
RESPONSE.setStatus(204)
return RESPONSE
def _appendData(self, data_chunk, content_type=None):
"""append data chunk to the end of the file
NOTE if content_type is specified, it will change content_type for the
whole file.
"""
data, size = self._read_data(data_chunk, data=self._baseGetData())
content_type=self._get_content_type(data_chunk, data, self.__name__,
content_type or self.content_type)
self.update_data(data, content_type, size)
# CMFFile also brings the IContentishInterface on CMF 2.2, remove it.
removeIContentishInterface(BigFile)