Commit d34b5f8c authored by Ekaterina's avatar Ekaterina

merge: with upstream nexedi/wendelin.git

parents bce74d9d 821ec8dc
Releases
===
1.0.162, 15.09.2010
---
- Data Lake uses same unified ingestion model as other agents
- default storage policy switched from "append as string" to MsgPack format
- improvements in Data Lake UI
- security tests added for Data Lake
...@@ -8,6 +8,14 @@ Wendelin combines the performance of scikit-learn machine learning library with ...@@ -8,6 +8,14 @@ Wendelin combines the performance of scikit-learn machine learning library with
NEO DB distributed storage in order to provide out-of-core processing of large NEO DB distributed storage in order to provide out-of-core processing of large
data sets. data sets.
Additional source code of components part of Wendelin project
-----------
- https://lab.nexedi.com/nexedi/wendelin.core - wendelin.core (also in pypi) - Out-of-Core processing module
- https://lab.nexedi.com/nexedi/fluent-plugin-wendelin - fluentd Wendelin plugin
- https://lab.nexedi.com/nexedi/fluent-plugin-bin - fluentd binary Wendelin plugin
- https://lab.nexedi.com/nexedi/ebulk - ebulk Wendelin plugin
Main applications Main applications
----------- -----------
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -50,6 +50,10 @@ ...@@ -50,6 +50,10 @@
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -50,6 +50,10 @@ ...@@ -50,6 +50,10 @@
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -2,13 +2,4 @@ ...@@ -2,13 +2,4 @@
<item key="sql_catalog_object_list" type="int"> <item key="sql_catalog_object_list" type="int">
<value>1</value> <value>1</value>
</item> </item>
<item key="_is_filtered_archive" type="int">
<value>1</value>
</item>
<item key="_filter_expression_archive" type="str">
<value>python: context.getPortalType() == 'Data Analysis'</value>
</item>
<item key="_filter_expression_cache_key_archive" type="tuple">
<value>portal_type</value>
</item>
</catalog_method> </catalog_method>
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -39,6 +39,22 @@ getRefreshState</string> </value> ...@@ -39,6 +39,22 @@ getRefreshState</string> </value>
<key> <string>connection_id</string> </key> <key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value> <value> <string>erp5_sql_connection</string> </value>
</item> </item>
<item>
<key> <string>expression</string> </key>
<value> <string>python: context.getPortalType() == \'Data Analysis\'</string> </value>
</item>
<item>
<key> <string>expression_cache_key</string> </key>
<value>
<tuple>
<string>portal_type</string>
</tuple>
</value>
</item>
<item>
<key> <string>filtered</string> </key>
<value> <int>1</int> </value>
</item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z_catalog_data_analysis_list</string> </value> <value> <string>z_catalog_data_analysis_list</string> </value>
...@@ -51,10 +67,20 @@ getRefreshState</string> </value> ...@@ -51,10 +67,20 @@ getRefreshState</string> </value>
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
</item> </item>
<item>
<key> <string>type</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary> </dictionary>
</pickle> </pickle>
</record> </record>
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -56,6 +56,10 @@ ...@@ -56,6 +56,10 @@
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -50,6 +50,10 @@ ...@@ -50,6 +50,10 @@
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -54,6 +54,10 @@ query_table="catalog"</string> </value> ...@@ -54,6 +54,10 @@ query_table="catalog"</string> </value>
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -55,6 +55,10 @@ query_table="catalog"</string> </value> ...@@ -55,6 +55,10 @@ query_table="catalog"</string> </value>
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/> <global name="SQL Method" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -55,6 +55,10 @@ query_table="catalog"</string> </value> ...@@ -55,6 +55,10 @@ query_table="catalog"</string> </value>
<key> <string>max_rows_</string> </key> <key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value> <value> <int>1000</int> </value>
</item> </item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
......
...@@ -28,9 +28,9 @@ ...@@ -28,9 +28,9 @@
############################################################################## ##############################################################################
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5.Document.BigFile import BigFile from erp5.component.document.BigFile import BigFile
from wendelin.bigarray.array_zodb import ZBigArray from wendelin.bigarray.array_zodb import ZBigArray
from Products.ERP5.Document.File import _MARKER from erp5.component.document.File import _MARKER
from ZPublisher import HTTPRangeSupport from ZPublisher import HTTPRangeSupport
from webdav.common import rfc1123_date from webdav.common import rfc1123_date
from DateTime import DateTime from DateTime import DateTime
...@@ -239,10 +239,11 @@ class DataArray(BigFile): ...@@ -239,10 +239,11 @@ class DataArray(BigFile):
RESPONSE.setHeader('Content-Range', RESPONSE.setHeader('Content-Range',
'bytes %d-%d/%d' % (start, end - 1, self.getSize())) 'bytes %d-%d/%d' % (start, end - 1, self.getSize()))
RESPONSE.setStatus(206) # Partial content RESPONSE.setStatus(206) # Partial content
self.log(str((start, end)))
# convert array to bytes # convert array to bytes
data = array[:].view('<b').reshape((-1,))[start:end].tobytes() data = array[:].view('<b').reshape((-1,))[start:end].tobytes()
RESPONSE.setBody(data, lock=True) RESPONSE.setBody(data, lock=True)
else: else:
boundary = choose_boundary() boundary = choose_boundary()
...@@ -268,6 +269,7 @@ class DataArray(BigFile): ...@@ -268,6 +269,7 @@ class DataArray(BigFile):
'multipart/%sbyteranges; boundary=%s' % ( 'multipart/%sbyteranges; boundary=%s' % (
draftprefix, boundary)) draftprefix, boundary))
RESPONSE.setStatus(206) # Partial content RESPONSE.setStatus(206) # Partial content
data = '' data = ''
for start, end in ranges: for start, end in ranges:
data = '{data}\r\n--{boundary}\r\n'\ data = '{data}\r\n--{boundary}\r\n'\
...@@ -283,4 +285,4 @@ class DataArray(BigFile): ...@@ -283,4 +285,4 @@ class DataArray(BigFile):
data = '{}\r\n--{}--\r\n'.format(data, boundary) data = '{}\r\n--{}--\r\n'.format(data, boundary)
RESPONSE.setBody(data, lock=True) RESPONSE.setBody(data, lock=True)
return True return True
\ No newline at end of file
...@@ -100,24 +100,28 @@ ...@@ -100,24 +100,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
...@@ -102,24 +102,28 @@ ...@@ -102,24 +102,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
...@@ -30,10 +30,10 @@ import hashlib ...@@ -30,10 +30,10 @@ import hashlib
from BTrees.OOBTree import OOBTree from BTrees.OOBTree import OOBTree
from BTrees.LOBTree import LOBTree from BTrees.LOBTree import LOBTree
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5.Document.Document import Document from erp5.component.document.Document import Document
from Products.ERP5Type import Permissions, PropertySheet from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5Type.BTreeData import PersistentString from erp5.component.module.BTreeData import PersistentString
from Products.ERP5Type.Log import log from erp5.component.module.Log import log
class IndexSequence: class IndexSequence:
""" """
......
...@@ -103,24 +103,28 @@ ...@@ -103,24 +103,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
...@@ -26,9 +26,12 @@ ...@@ -26,9 +26,12 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import msgpack
import struct
import numpy as np
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5.Document.BigFile import BigFile from erp5.component.document.BigFile import BigFile
class DataStream(BigFile): class DataStream(BigFile):
""" """
...@@ -57,6 +60,33 @@ class DataStream(BigFile): ...@@ -57,6 +60,33 @@ class DataStream(BigFile):
for chunk in data.iterate(start_offset, end_offset - start_offset): for chunk in data.iterate(start_offset, end_offset - start_offset):
yield chunk yield chunk
def readMsgpackChunkList(self, start_offset, end_offset):
"""
Read chunks of msgpack data from a Data Stream and return (unpacked_data list, offset)
"""
unpacker = msgpack.Unpacker()
data = self._baseGetData()
pos = start_offset
data_list = []
for chunk in data.iterate(start_offset, end_offset - start_offset):
unpacker.feed(chunk)
while True:
pos = start_offset + unpacker.tell()
try:
#yield unpacker.unpack()
data_list.append(unpacker.unpack())
except msgpack.exceptions.OutOfData:
break
#raise StopIteration(pos)
return data_list, pos
def extractDateTime(self, date_time_holder):
if isinstance(date_time_holder, int):
return np.datetime64(date_time_holder, 's')
# if it is not in, we Expect msgpack.ExtType
s, ns = struct.unpack(">II", date_time_holder.data)
return np.datetime64(s, 's') + np.timedelta64(ns, 'ns')
def readChunkList(self, start_offset, end_offset): def readChunkList(self, start_offset, end_offset):
""" """
Read chunks of data from a Data Stream and return them. Read chunks of data from a Data Stream and return them.
...@@ -67,4 +97,27 @@ class DataStream(BigFile): ...@@ -67,4 +97,27 @@ class DataStream(BigFile):
for chunk in data.iterate(start_offset, end_offset - start_offset): for chunk in data.iterate(start_offset, end_offset - start_offset):
chunk_list.append(chunk) chunk_list.append(chunk)
return chunk_list return chunk_list
\ No newline at end of file
def getRecursiveSuccessorValueList(self):
"""
Return list of all successors (Data Streams) for a Data Stream.
"""
successor_list = []
successor = self.getSuccessorValue()
while successor is not None:
successor_list.append(successor)
successor = successor.getSuccessorValue()
return successor_list
def getRecursivePredecessorValueList(self):
"""
Return list of all predecessor (Data Streams) for a Data Stream.
"""
predecessor_list = []
predecessor = self.getPredecessorValue()
while predecessor is not None:
predecessor_list.append(predecessor)
predecessor = predecessor.getPredecessorValue()
return predecessor_list
...@@ -45,9 +45,7 @@ ...@@ -45,9 +45,7 @@
<item> <item>
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple/>
<string>W: 71, 2: Arguments number differs from overridden \'appendData\' method (arguments-differ)</string>
</tuple>
</value> </value>
</item> </item>
<item> <item>
...@@ -102,24 +100,28 @@ ...@@ -102,24 +100,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
# Klaus Wölfel <klaus@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from erp5.component.document.Amount import Amount
from erp5.component.document.Transformation import Transformation
class DataTransformation(Transformation):
"""
Contains a list of transformed data operations and the data operation used
"""
meta_type = 'ERP5 Data Transformation'
portal_type = 'Data Transformation'
# Use Amount implementation of *VariationCategoryList
# instead of the one VariatedMixin used by Transformation so that we can set Variation
# on Initial Product
getVariationBaseCategoryList = Amount.getVariationBaseCategoryList
getVariationBaseCategoryItemList = Amount.getVariationBaseCategoryItemList
getVariationCategoryList = Amount.getVariationCategoryList
getVariationCategoryItemList = Amount.getVariationCategoryItemList
getVariationRangeBaseCategoryList = Amount.getVariationRangeBaseCategoryList
getVariationRangeCategoryList = Amount.getVariationRangeCategoryList
getVariationRangeCategoryItemList = Amount.getVariationRangeCategoryItemList
setVariationBaseCategoryList = Amount.setVariationBaseCategoryList
setVariationCategoryList = Amount.setVariationCategoryList
_setVariationCategoryList = Amount._setVariationCategoryList
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="Extension Component" module="erp5.portal_type"/> <global name="Document Component" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -14,7 +14,11 @@ ...@@ -14,7 +14,11 @@
</item> </item>
<item> <item>
<key> <string>default_reference</string> </key> <key> <string>default_reference</string> </key>
<value> <string>external_process_raw_data</string> </value> <value> <string>DataTransformation</string> </value>
</item>
<item>
<key> <string>default_source_reference</string> </key>
<value> <string>Products.ERP5.Document.Transformation</string> </value>
</item> </item>
<item> <item>
<key> <string>description</string> </key> <key> <string>description</string> </key>
...@@ -24,17 +28,11 @@ ...@@ -24,17 +28,11 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>extension.erp5.external_process_raw_data</string> </value> <value> <string>document.erp5.DataTransformation</string> </value>
</item>
<item>
<key> <string>language</string> </key>
<value>
<none/>
</value>
</item> </item>
<item> <item>
<key> <string>portal_type</string> </key> <key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value> <value> <string>Document Component</string> </value>
</item> </item>
<item> <item>
<key> <string>sid</string> </key> <key> <string>sid</string> </key>
...@@ -51,15 +49,7 @@ ...@@ -51,15 +49,7 @@
<item> <item>
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple/>
<string>W: 74, 2: No exception type(s) specified (bare-except)</string>
<string>W: 79, 4: No exception type(s) specified (bare-except)</string>
<string>W: 95, 8: Unused variable \'times\' (unused-variable)</string>
<string>W:131, 6: No exception type(s) specified (bare-except)</string>
<string>W:166, 8: Unreachable code (unreachable)</string>
<string>W:184, 11: Using type() instead of isinstance() for a typecheck. (unidiomatic-typecheck)</string>
<string>W:188, 10: No exception type(s) specified (bare-except)</string>
</tuple>
</value> </value>
</item> </item>
<item> <item>
...@@ -114,24 +104,28 @@ ...@@ -114,24 +104,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
...@@ -98,24 +98,28 @@ ...@@ -98,24 +98,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
...@@ -100,24 +100,28 @@ ...@@ -100,24 +100,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
...@@ -100,24 +100,28 @@ ...@@ -100,24 +100,28 @@
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_acquisition_unit_module</id> <id>data_acquisition_unit_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_aggregation_unit_module</id> <id>data_aggregation_unit_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_analysis_module</id> <id>data_analysis_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_array_module</id> <id>data_array_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_configuration_module</id> <id>data_configuration_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_descriptor_module</id> <id>data_descriptor_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_event_module</id> <id>data_event_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_ingestion_batch_module</id> <id>data_ingestion_batch_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_ingestion_module</id> <id>data_ingestion_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_license_module</id> <id>data_license_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_operation_module</id> <id>data_operation_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_order_module</id> <id>data_order_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_product_module</id> <id>data_product_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_release_module</id> <id>data_release_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_set_module</id> <id>data_set_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_stream_module</id> <id>data_stream_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_supply_module</id> <id>data_supply_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_transformation_module</id> <id>data_transformation_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>device_configuration_module</id> <id>device_configuration_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<module> <module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>progress_indicator_module</id> <id>progress_indicator_module</id>
<permission_list> <permission_list>
<permission type='tuple'> <permission type='tuple'>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Operation" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>version</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>data_operation_script_id</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>use_list</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>lines</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>quantity_unit_list</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>lines</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>aggregated_portal_type_list</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>lines</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>base_contribution_list</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>lines</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Acquisition Unit</string>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type_list</string> </key>
<value>
<tuple>
<string>Data Stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>base_contribution_list</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>quantity_unit/unit/piece</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ingest-fluent-data</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>This data operation can be used to ingest data with fluentd. It assumes data comes in msgpack format.\n
It will first unpack the msgpack, then remove the timestamp, and convert the data part (usually a dictionary)\n
to string and append it to Data Stream.\n
\n
Note that what is saved to Data Stream might be different from what fluentd was reading initially,\n
depending on fluentd plugin configuration. For example fluentd might convert json to msgpack then\n
what is saved in Data Stream might be a string representation of a python dictionary and not json.</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>wendelin_ingest_data_conversion</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Data Operation</string> </value>
</item>
<item>
<key> <string>quantity_unit_list</string> </key>
<value>
<tuple>
<string>information/byte</string>
</tuple>
</value>
</item>
<item>
<key> <string>reference</string> </key>
<value> <string>FOURIER-MAX</string> </value>
</item>
<item>
<key> <string>script_id</string> </key>
<value> <string>DataIngestionLine_writeFluentdIngestionToDataStream</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Ingest Fluentd Data</string> </value>
</item>
<item>
<key> <string>use_list</string> </key>
<value>
<tuple>
<string>big_data/analysis</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>001</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Product" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Stream</string>
<string>Progress Indicator</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>test_product</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>default</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Data Product</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Append to Data Stream</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Supply Line" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>resource/data_operation_module/wendelin_ingest_data</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ingestion_operation</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>1</string> </value>
</item>
<item>
<key> <string>int_index</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Data Supply Line</string> </value>
</item>
<item>
<key> <string>quantity</string> </key>
<value> <float>1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Ingest Data</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>edit_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>0.0.0.0</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1596024084.33</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1596024084.33</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>draft</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
This script is used during fluentd ingestion.
It assumes data comes in msgpack encoded in the following format: mspack(timestamp, data).
It will first unpack the msgpack, then remove the first item of the tuple (timestamp) and
append str(data) to "Data Stream".
Note that what is saved to Data Stream might be different from what fluentd was reading
initially, depending on fluentd plugin configuration. For example fluentd might convert
json to msgpack, then what is saved in Data Stream might be str(python_dict) and not json.
"""
out_stream["Data Stream"].appendData(''.join([str(c[1]) for c in context.unpack(data_chunk)])) out_stream["Data Stream"].appendData(''.join([str(c[1]) for c in context.unpack(data_chunk)]))
"""
This script is a general ingestion script which can be used with fluentd or with other http based ingestion tools.
It will append data sent to Wendelin 'as it is' to respective "Data Stream".
Note that by default fluentd data is encoded in msgpack format, this script will not unpack it.
"""
out_stream["Data Stream"].appendData(data_chunk)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PyData Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>comment</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>data_chunk, out_stream={}</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string>This script is used with default simple Wendelin model defined in erp5_wendelin_data.\n
For other models it will need adjustments.</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Ingestion operation script to be used with Fluentd Input Bin plugin: Append given data chunk to given data stream</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataIngestionLine_writeIngestionToDataStream</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>PyData Script</string> </value>
</item>
<item>
<key> <string>reference</string> </key>
<value> <string>DataIngestionLine_writeIngestionToDataStream</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>DataIngestionLine_writeIngestionToDataStream</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
This script is used in Ingestion Policy objects to parse provided by fluentd / ebulk reference.
In this case it expects a reference in this format for example:
'sensor1'
i.e. no special characters and one name only.
If you need to use more complex hierachical reference notations like:
'sensor1.product1'
you can use script: 'IngestionPolicy_parseSimpleIngestionTag'
"""
project_reference = reference.split('.')[0] project_reference = reference.split('.')[0]
return {'resource_reference' : reference, return {'resource_reference' : reference,
'specialise_reference': project_reference, 'specialise_reference': project_reference,
......
"""
This script is used in Ingestion Policy objects to parse provided by fluentd / ebulk reference.
In this case it expects a reference in this format for example:
'sensor1.product1'
"""
sensor_reference = reference.split('.')[0]
data_product_reference = reference.split('.')[1]
return {'resource_reference' : data_product_reference,
'specialise_reference': sensor_reference,
'reference': sensor_reference }
...@@ -62,22 +62,34 @@ ...@@ -62,22 +62,34 @@
<value> <string>string</string> </value> <value> <string>string</string> </value>
</item> </item>
</dictionary> </dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>comment</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple> </tuple>
</value> </value>
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>input_stream_data, output_array, output_descriptor</string> </value> <value> <string>reference</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string>tag example: sensor_1.sample-data</string> </value>
</item> </item>
<item> <item>
<key> <string>description</string> </key> <key> <string>description</string> </key>
<value> <value> <string>tag example: sensor_1.sample-data</string> </value>
<none/>
</value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>DataTransformation_transformFIFDataToDataArray</string> </value> <value> <string>IngestionPolicy_parseSimpleIngestionTag</string> </value>
</item> </item>
<item> <item>
<key> <string>portal_type</string> </key> <key> <string>portal_type</string> </key>
...@@ -85,7 +97,11 @@ ...@@ -85,7 +97,11 @@
</item> </item>
<item> <item>
<key> <string>reference</string> </key> <key> <string>reference</string> </key>
<value> <string>DataTransformation_transformFIFDataToDataArray</string> </value> <value> <string>IngestionPolicy_parseSimpleIngestionTag</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>IngestionPolicy_parseSimpleIngestionTag</string> </value>
</item> </item>
</dictionary> </dictionary>
</pickle> </pickle>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Base Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>acquisition_append_value</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>acquisition_base_category</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>acquisition_copy_value</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>acquisition_mask_value</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>acquisition_object_id</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>acquisition_portal_type</string> </key>
<value> <string>python:[]</string> </value>
</item>
<item>
<key> <string>acquisition_sync_value</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>business_application</string>
</tuple>
</value>
</item>
<item>
<key> <string>category_type</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>criterion_property</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>fallback_base_category</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>business_application</string> </value>
</item>
<item>
<key> <string>id_generator</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id_group</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>membership_criterion_base_category</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>membership_criterion_category</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>multimembership_criterion_base_category</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Base Category</string> </value>
</item>
<item>
<key> <string>read_permission</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>rid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>short_title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>test_method_id</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Business Application</string> </value>
</item>
<item>
<key> <string>write_permission</string> </key>
<value>
<none/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>base</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Base</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>big_data</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Big Data</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>crm</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>CRM</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>dms</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>DMS</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>pdm</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>PDM</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>project_management</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Project Management</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>trade</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Trade</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Base Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>category_type</string> </key>
<value>
<tuple>
<string>variation</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>resolution</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Base Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Resolution</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>data</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>convert</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Convert</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>subsample</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Subsample</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Category" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>use/big_data/ingestion</string>
<string>use/big_data/ingestion</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ingestion</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Category</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Ingestion</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment