Commit 8035615a authored by Ivan Tyagov's avatar Ivan Tyagov

WIP: tests using new model.

parent d416f797
data_stream = out_data_stream['Data Stream']
data_stream.appendData(''.join([c[1] for c in context.unpack(data_chunk)]))
......@@ -77,13 +77,12 @@
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>data_chunk=None, data_stream=None</string> </value>
<value> <string>data_chunk=None, data_stream=None, bucket_reference=None, out_data_stream=None</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
<value> <string>This script is used with default simple Wendelin model defined in erp5_wendelin_data.\n
For other models it will need adjustments.</string> </value>
</item>
<item>
<key> <string>description</string> </key>
......
......@@ -28,8 +28,8 @@
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import createZODBPythonScript
from wendelin.bigarray.array_zodb import ZBigArray
from DateTime import DateTime
from zExceptions import NotFound
#from DateTime import DateTime
#from zExceptions import NotFound
import msgpack
import numpy as np
import string
......@@ -52,28 +52,6 @@ class Test(ERP5TypeTestCase):
def getTitle(self):
return "Wendelin Test"
def afterSetUp(self):
"""
This is ran before anything, used to set the environment
"""
# here, you can create the categories and objects your test will depend on
pass
def stepSetupIngestion(self, reference):
"""
Generic step.
"""
ingestion_policy, data_supply, data_stream, data_array = \
self.portal.portal_ingestion_policies.IngestionPolicyTool_addIngestionPolicy( \
reference = reference, \
batch_mode = 1)
# to avoid random test failures due to test execution we make start date one day before
data_supply.setStartDate(DateTime() - 1)
self.tic()
return ingestion_policy, data_supply, data_stream, data_array
def test_0_import(self):
"""
Test we can import certain libraries but still failure to do so should be a
......@@ -92,7 +70,9 @@ class Test(ERP5TypeTestCase):
portal = self.portal
request = portal.REQUEST
reference = getRandomString()
ingestion_policy = portal.restrictedTraverse("portal_ingestion_policies/wendelin_1")
data_supply = portal.restrictedTraverse("data_supply_module/wendelin_1")
reference = 'wendelin-default-ingestion'
number_string_list = []
for my_list in list(chunks(range(0, 100001), 10)):
number_string_list.append(','.join([str(x) for x in my_list]))
......@@ -100,19 +80,27 @@ class Test(ERP5TypeTestCase):
# make sure real_data tail is also a full line
real_data += '\n'
ingestion_policy, _, data_stream, data_array = \
self.stepSetupIngestion(reference)
# simulate fluentd by setting proper values in REQUEST
request.method = 'POST'
data_chunk = msgpack.packb([0, real_data], use_bin_type=True)
request.set('reference', reference)
request.set('data_chunk', data_chunk)
ingestion_policy.ingest()
self.tic()
# get related Data ingestion
data_ingestion = data_supply.Base_getRelatedObjectList(portal_type='Data Ingestion')[0]
self.assertNotEqual(None, data_ingestion)
data_ingestion_line = [x for x in data_ingestion.objectValues() if x.getReference() == 'out_data_stream'][0]
data_stream = data_ingestion_line.getAggregateValue()
self.assertEqual('Data Stream', data_stream.getPortalType())
data_stream_data = data_stream.getData()
self.assertEqual(real_data, data_stream_data)
return
"""
# try sample transformation
data_stream.DataStream_transform(\
chunk_length = 10450, \
......@@ -129,6 +117,8 @@ class Test(ERP5TypeTestCase):
# test ingesting with bad reference and raise of NotFound
request.set('reference', reference + 'not_existing')
self.assertRaises(NotFound, ingestion_policy.ingest)
"""
def test_01_1_IngestionTail(self):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment