Skip to content

erp5
erp5
Commit 4f3bb0c9 authored by Kirill Smelkov's avatar Kirill Smelkov
3 Browse files
Options

Revert "BigFile: Fixes, Tests and on-server Append support" 

This reverts commit 193f5cdd, reversing
changes made to 4ee61a23.

Jean-Paul suggested we first better further review our code review / merging
procedures and use e.g. this particular merge request as a basis for that.

Thus I'm reverting this merge, so people could study and re-merge it
"properly".

~~~~

Please note: I could potentially rewrite master history so that there
would be a no merge commit at all, as e.g. my branch was not so far ever
merged at all. In contrast to draft branches, that is however a not good
practice to rebase, and thus rewrite, master history - what has been
committed is committed and we only continue.

So later to re-merge my branch, if it will not be changed, we'll need to
first revert this revert (see [1] for rationale), or alternatively I
could re-prepare my patches with different sha1 ids and they will merge
again the "usual way".

[1] http://git-scm.com/docs/howto/revert-a-faulty-merge.html
parent e934b59f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 39 additions and 557 deletions
bt5/erp5_big_file/SkinTemplateItem/portal_skins/erp5_big_file/BigFileModule_viewBigFileList.xml
View file @ 4f3bb0c9
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="ERP5 Form" module="erp5.portal_type"/> <global name="ERP5Form" module="Products.ERP5Form.Form"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
......
bt5/erp5_big_file/SkinTemplateItem/portal_skins/erp5_big_file/BigFile_view.xml
View file @ 4f3bb0c9
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="ERP5 Form" module="erp5.portal_type"/> <global name="ERP5Form" module="Products.ERP5Form.Form"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
......
bt5/erp5_big_file/TestTemplateItem/portal_components/test.erp5.testBigFile.xml deleted 100644 → 0
View file @ e934b59f
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Test Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>testBigFile</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>test.erp5.testBigFile</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Test Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
bt5/erp5_big_file/bt/template_test_id_list deleted 100644 → 0
View file @ e934b59f
test.erp5.testBigFile
\ No newline at end of file
bt5/erp5_big_file/bt/test_dependency_list deleted 100644 → 0
View file @ e934b59f
erp5_full_text_mroonga_catalog
\ No newline at end of file
bt5/erp5_big_file/bt/version
View file @ 4f3bb0c9
1.1 1
\ No newline at end of file \ No newline at end of file
product/ERP5/Document/BigFile.py
View file @ 4f3bb0c9
...@@ -24,42 +24,14 @@ from ZPublisher.HTTPRequest import FileUpload ...@@ -24,42 +24,14 @@ from ZPublisher.HTTPRequest import FileUpload
from ZPublisher import HTTPRangeSupport from ZPublisher import HTTPRangeSupport
from webdav.common import rfc1123_date from webdav.common import rfc1123_date
from mimetools import choose_boundary from mimetools import choose_boundary
from Products.CMFCore.utils import _setCacheHeaders, _ViewEmulator from Products.CMFCore.utils import getToolByName, _setCacheHeaders,\
from DateTime import DateTime _ViewEmulator
import re import re
class BigFile(File): class BigFile(File):
""" """
Support storing huge file. Support storing huge file.
No convertion is allowed for now. No convertion is allowed for now.
NOTE BigFile maintains the following invariant:
data property is either
- BTreeData instance, or
- str(*), or
- None.
(*) str has to be supported because '' is a default value for `data` field
from Data property sheet.
Even more - for
a) compatibility reasons, and
b) desire to support automatic migration of File-based documents
from document_module to BigFiles
non-empty str for data also have to be supported.
XXX(kirr) I'm not sure supporting non-empty str is a good idea (it
would be simpler if .data could be either BTreeData or "empty"),
but neither I'm experienced enough in erp5 nor know what are
appropriate compatibility requirements.
We discussed with Romain and settled on "None or str or BTreeData"
invariant for now.
""" """
meta_type = 'ERP5 Big File' meta_type = 'ERP5 Big File'
...@@ -126,11 +98,6 @@ class BigFile(File): ...@@ -126,11 +98,6 @@ class BigFile(File):
if data is None: if data is None:
btree = BTreeData() btree = BTreeData()
elif isinstance(data, str):
# we'll want to append content to this file -
# - automatically convert str (empty or not) to BTreeData
btree = BTreeData()
btree.write(data, 0)
else: else:
btree = data btree = data
seek(0) seek(0)
...@@ -149,14 +116,6 @@ class BigFile(File): ...@@ -149,14 +116,6 @@ class BigFile(File):
self.serialize() self.serialize()
return btree, len(btree) return btree, len(btree)
def _data_mtime(self):
"""get .data mtime if present and fallback to self._p_mtime"""
# there is no data._p_mtime when data is None or str.
# so try and fallback to self._p_mtime
data = self._baseGetData()
mtime = getattr(data, '_p_mtime', self._p_mtime)
return mtime
def _range_request_handler(self, REQUEST, RESPONSE): def _range_request_handler(self, REQUEST, RESPONSE):
# HTTP Range header handling: return True if we've served a range # HTTP Range header handling: return True if we've served a range
# chunk out of our data. # chunk out of our data.
...@@ -188,10 +147,13 @@ class BigFile(File): ...@@ -188,10 +147,13 @@ class BigFile(File):
try: mod_since=long(DateTime(date).timeTime()) try: mod_since=long(DateTime(date).timeTime())
except: mod_since=None except: mod_since=None
if mod_since is not None: if mod_since is not None:
last_mod = self._data_mtime() if data is not None:
if last_mod is None: last_mod = long(data._p_mtime)
last_mod = 0 else:
last_mod = long(last_mod) if self._p_mtime:
last_mod = long(self._p_mtime)
else:
last_mod = long(0)
if last_mod > mod_since: if last_mod > mod_since:
# Modified, so send a normal response. We delete # Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200 # the ranges, which causes us to skip to the 200
...@@ -210,7 +172,10 @@ class BigFile(File): ...@@ -210,7 +172,10 @@ class BigFile(File):
RESPONSE.setHeader('Content-Range', RESPONSE.setHeader('Content-Range',
'bytes */%d' % self.getSize()) 'bytes */%d' % self.getSize())
RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime())) if data is not None:
RESPONSE.setHeader('Last-Modified', rfc1123_date(data._p_mtime))
else:
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._p_mtime))
RESPONSE.setHeader('Content-Type', self.content_type) RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', self.getSize()) RESPONSE.setHeader('Content-Length', self.getSize())
RESPONSE.setStatus(416) RESPONSE.setStatus(416)
...@@ -223,7 +188,10 @@ class BigFile(File): ...@@ -223,7 +188,10 @@ class BigFile(File):
start, end = ranges[0] start, end = ranges[0]
size = end - start size = end - start
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime())) if data is not None:
RESPONSE.setHeader('Last-Modified', rfc1123_date(data._p_mtime))
else:
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._p_mtime))
RESPONSE.setHeader('Content-Type', self.content_type) RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', size) RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setHeader('Accept-Ranges', 'bytes')
...@@ -231,7 +199,6 @@ class BigFile(File): ...@@ -231,7 +199,6 @@ class BigFile(File):
'bytes %d-%d/%d' % (start, end - 1, self.getSize())) 'bytes %d-%d/%d' % (start, end - 1, self.getSize()))
RESPONSE.setStatus(206) # Partial content RESPONSE.setStatus(206) # Partial content
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str): if isinstance(data, str):
RESPONSE.write(data[start:end]) RESPONSE.write(data[start:end])
return True return True
...@@ -260,7 +227,10 @@ class BigFile(File): ...@@ -260,7 +227,10 @@ class BigFile(File):
RESPONSE.setHeader('Content-Length', size) RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime())) if data is not None:
RESPONSE.setHeader('Last-Modified', rfc1123_date(data._p_mtime))
else:
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._p_mtime))
RESPONSE.setHeader('Content-Type', RESPONSE.setHeader('Content-Type',
'multipart/%sbyteranges; boundary=%s' % ( 'multipart/%sbyteranges; boundary=%s' % (
draftprefix, boundary)) draftprefix, boundary))
...@@ -274,7 +244,6 @@ class BigFile(File): ...@@ -274,7 +244,6 @@ class BigFile(File):
'Content-Range: bytes %d-%d/%d\r\n\r\n' % ( 'Content-Range: bytes %d-%d/%d\r\n\r\n' % (
start, end - 1, self.getSize())) start, end - 1, self.getSize()))
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str): if isinstance(data, str):
RESPONSE.write(data[start:end]) RESPONSE.write(data[start:end])
...@@ -311,7 +280,7 @@ class BigFile(File): ...@@ -311,7 +280,7 @@ class BigFile(File):
data = self._baseGetData() data = self._baseGetData()
mime = self.getContentType() mime = self.getContentType()
RESPONSE.setHeader('Content-Length', data is not None and len(data) or 0) RESPONSE.setHeader('Content-Length', len(data))
RESPONSE.setHeader('Content-Type', mime) RESPONSE.setHeader('Content-Type', mime)
if inline is _MARKER: if inline is _MARKER:
# by default, use inline for text and image formats # by default, use inline for text and image formats
...@@ -344,8 +313,7 @@ class BigFile(File): ...@@ -344,8 +313,7 @@ class BigFile(File):
content_range = REQUEST.get_header('Content-Range', None) content_range = REQUEST.get_header('Content-Range', None)
if content_range is None: if content_range is None:
# truncate the file btree = None
self._baseSetData(None)
else: else:
current_size = int(self.getSize()) current_size = int(self.getSize())
query_range = re.compile('bytes \*/\*') query_range = re.compile('bytes \*/\*')
...@@ -353,6 +321,8 @@ class BigFile(File): ...@@ -353,6 +321,8 @@ class BigFile(File):
'(?P<last_byte>[0-9]+)/' \ '(?P<last_byte>[0-9]+)/' \
'(?P<total_content_length>[0-9]+)') '(?P<total_content_length>[0-9]+)')
if query_range.match(content_range): if query_range.match(content_range):
data = self._baseGetData()
RESPONSE.setHeader('X-Explanation', 'Resume incomplete') RESPONSE.setHeader('X-Explanation', 'Resume incomplete')
RESPONSE.setHeader('Range', 'bytes 0-%s' % (current_size-1)) RESPONSE.setHeader('Range', 'bytes 0-%s' % (current_size-1))
RESPONSE.setStatus(308) RESPONSE.setStatus(308)
...@@ -379,28 +349,25 @@ class BigFile(File): ...@@ -379,28 +349,25 @@ class BigFile(File):
RESPONSE.setStatus(400) RESPONSE.setStatus(400)
return RESPONSE return RESPONSE
else:
btree = self._baseGetData()
if btree is None:
btree = BTreeData()
else: else:
RESPONSE.setHeader('X-Explanation', 'Can not parse range') RESPONSE.setHeader('X-Explanation', 'Can not parse range')
RESPONSE.setStatus(400) # Partial content RESPONSE.setStatus(400) # Partial content
return RESPONSE return RESPONSE
self._appendData(file, content_type=type) data, size = self._read_data(file, data=btree)
RESPONSE.setStatus(204) content_type=self._get_content_type(file, data, self.__name__,
return RESPONSE type or self.content_type)
def _appendData(self, data_chunk, content_type=None):
"""append data chunk to the end of the file
NOTE if content_type is specified, it will change content_type for the
whole file.
"""
data, size = self._read_data(data_chunk, data=self._baseGetData())
content_type=self._get_content_type(data_chunk, data, self.__name__,
content_type or self.content_type)
self.update_data(data, content_type, size) self.update_data(data, content_type, size)
RESPONSE.setStatus(204)
return RESPONSE
# CMFFile also brings the IContentishInterface on CMF 2.2, remove it. # CMFFile also brings the IContentishInterface on CMF 2.2, remove it.
removeIContentishInterface(BigFile) removeIContentishInterface(BigFile)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7