Clean up.

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.py

-from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery
-
-return
-# This alarm was deprecated - kept for test
-
-portal = context.getPortalObject()
-portal_catalog = portal.portal_catalog
-
-# invalidate old (more than 5 hours) pending ingestions (e.g. split ingestions that were canceled/interrumped and no resumed)
-from DateTime import DateTime
-now = DateTime()
-now_minus_max = now - 1.0/24/60*9999
-now_minus_5 = now - 1.0/24/60*60*5
-catalog_kw = {'creation_date': {'query': (now_minus_max, now_minus_5), 'range': 'minmax'}, 'simulation_state': 'started', 'portal_type': 'Data Ingestion'}
-
-for data_ingestion in portal_catalog(**catalog_kw):
-  # search related data ingestions that are not old yet (less than 5 hours)
-  catalog_kw = {'creation_date': {'query': (now_minus_5, DateTime()), 'range': 'minmax'},
-                'simulation_state': 'started',
-                'portal_type': 'Data Ingestion',
-                'reference': data_ingestion.getReference()}
-  invalidate = True
-  if len(portal_catalog(**catalog_kw)) > 0:
-    invalidate = False
-
-  if invalidate:
-    # invalidate related Data Stream
-    kw_dict = {"portal_type": "Data Stream",
-               "id": data_ingestion.getId()}
-    for data_stream in portal_catalog(**kw_dict):
-      if not data_stream.getReference().endswith("_invalid"):
-        data_stream.setReference(data_stream.getReference() + "_invalid")
-        try:
-          data_stream.invalidate()
-        except:
-          context.logEntry("[WARNING] Could not invalidate data stream '%s', it was already invalidated or draft" % data_stream.getId())
-    try:
-      if not data_ingestion.getReference().endswith("_invalid"):
-        data_ingestion.setReference(data_ingestion.getReference() + "_invalid")
-      data_ingestion.deliver()
-    except:
-      context.logEntry("[WARNING] Could not invalidate/deliver data ingestion '%s', it was already invalidated/deliver" % data_ingestion.getId())

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
-        </item>
-        <item>
-            <key> <string>_params</string> </key>
-            <value> <string></string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>Alarm_dataLakeHandleInvalids</string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.py

-portal = context.getPortalObject()
-portal_catalog = portal.portal_catalog
-reference_separator = portal.getIngestionReferenceDictionary()["reference_separator"]
-none_extension = portal.getIngestionReferenceDictionary()["none_extension"]
-
-# check new reference
-data_ingestions = portal_catalog(portal_type = "Data Ingestion", reference = new_reference)
-if len(data_ingestions) > 0: raise "Error renaming: new reference '%s' already exists." % new_reference
-
-# rename data ingestions
-data_ingestions = portal_catalog(portal_type = "Data Ingestion", reference = reference)
-if len(data_ingestions) == 0: raise "Error renaming: could not find any data ingestion with reference '%s'." % reference
-data_ingestion_title = reference_separator.join(new_reference.split(reference_separator)[1:-1])
-for data_ingestion in data_ingestions:
-  data_ingestion.setReference(new_reference)
-  data_ingestion.setTitle(data_ingestion_title)
-
-extension = new_reference.split(reference_separator)[-1]
-data_stream_title = "%s%s" % (data_ingestion_title, "."+extension if extension != none_extension else "")
-# rename data streams
-data_streams = portal_catalog(portal_type = "Data Stream", reference = reference)
-for data_stream in data_streams:
-  data_stream.setReference(new_reference)
-  data_stream.setTitle(data_stream_title)
-
-# rename data analysis
-data_analysises = portal_catalog(portal_type = "Data Analysis", reference = reference)
-for data_analysis in data_analysises:
-  data_analysis.setReference(new_reference)
-
-# rename data arrays
-data_arrays = portal_catalog(portal_type = "Data Array", reference = reference)
-for data_array in data_arrays:
-  data_array.setReference(new_reference)

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
-        </item>
-        <item>
-            <key> <string>_params</string> </key>
-            <value> <string>reference, new_reference</string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>ERP5Site_renameIngestion</string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>