Commit bde87c2e authored by Iliya Manolov's avatar Iliya Manolov

Fixed DataStreams reindexing when appending new data and added a test...

parent d2867d16
...@@ -5,6 +5,23 @@ ...@@ -5,6 +5,23 @@
from wendelin.bigarray.array_zodb import ZBigArray from wendelin.bigarray.array_zodb import ZBigArray
import numpy as np import numpy as np
from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
def Base_denyObjectReindexation(self):
tv = getTransactionalVariable()
key = 'explicitly_deny_object_reindexation_list'
try:
tmp = tv[key]
if not isinstance(tmp, list):
tv[key] = []
except KeyError:
tv[key] = []
tv[key].append(self)
def DataStream_copyCSVToDataArray(data_stream, chunk_list, start, end, \ def DataStream_copyCSVToDataArray(data_stream, chunk_list, start, end, \
data_array_reference=None): data_array_reference=None):
""" """
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>Base_denyObjectReindexation</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>Wendelin</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_denyObjectReindexation</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -34,6 +34,7 @@ import msgpack ...@@ -34,6 +34,7 @@ import msgpack
import numpy as np import numpy as np
import string import string
import random import random
import transaction
def getRandomString(): def getRandomString():
return 'test_%s' %''.join([random.choice(string.ascii_letters + string.digits) \ return 'test_%s' %''.join([random.choice(string.ascii_letters + string.digits) \
...@@ -73,6 +74,24 @@ class Test(ERP5TypeTestCase): ...@@ -73,6 +74,24 @@ class Test(ERP5TypeTestCase):
return ingestion_policy, data_supply, data_stream, data_array return ingestion_policy, data_supply, data_stream, data_array
def countObjectReinstantiations(self, ob):
"""
Counts how many portal activity messages with a method_id of
immediateReindexObject have been called on ob.
"""
activity_tool = self.portal.portal_activities
activity_messages = activity_tool.getMessageList()
result = 0
last_messages = activity_messages[-10:]
for message in last_messages:
if message.getObject(activity_tool) == ob and message.method_id == 'immediateReindexObject':
result += 1
return result
def test_0_import(self): def test_0_import(self):
""" """
...@@ -341,3 +360,44 @@ context.activate().DataStream_readChunkListAndTransform( \ ...@@ -341,3 +360,44 @@ context.activate().DataStream_readChunkListAndTransform( \
# test as sequence # test as sequence
bucket = bucket_stream.getBucketItemSequence(start_key=10, count=1)[0] bucket = bucket_stream.getBucketItemSequence(start_key=10, count=1)[0]
self.assertEqual(100000, bucket[1].value) self.assertEqual(100000, bucket[1].value)
def test_05_NoReindexDataStream(self):
"""
Test if DataStreams reindex themselves when they have data appended to them.
"""
portal = self.portal
reference = getRandomString()
row = ','.join(['%s' %x for x in range(1000)])
number_string_list = [row]*20
real_data = '\n'.join(number_string_list)
data_stream = portal.data_stream_module.newContent(portal_type = 'Data Stream',
reference = reference)
self.tic()
data_stream.appendData(real_data)
transaction.commit()
result = self.countObjectReinstantiations(data_stream)
self.assertEquals(result, 0)
reference = getRandomString()
data_array = portal.data_array_module.newContent(portal_type = 'Data Array',
reference = reference)
self.tic()
data_array.edit(title='New title')
transaction.commit()
result = self.countObjectReinstantiations(data_array)
self.assertNotEquals(result, 0)
\ No newline at end of file
...@@ -79,7 +79,9 @@ ...@@ -79,7 +79,9 @@
<item> <item>
<key> <string>script_name</string> </key> <key> <string>script_name</string> </key>
<value> <value>
<tuple/> <list>
<string>DataStream_denyObjectReindexation</string>
</list>
</value> </value>
</item> </item>
<item> <item>
......
"""
Handle appended data chunks.
"""
data_stream = state_change['object']
data_stream.Base_denyObjectReindexation()
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>state_change, **kw</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStream_denyObjectReindexation</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment