Skip to content

erp5
erp5
Commit fa7ac3a5 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki
Browse files
Options

erp5_core: add ERP5Site_resynchroniseCatalogSince script.

parent 70b5d264
Pipeline #36643 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 272 additions and 0 deletions
    bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5Core.py
    View file @ fa7ac3a5
    ......@@ -28,6 +28,7 @@
    ##############################################################################
    import collections
    import csv
    import httplib
    import urlparse
    import base64
    ......@@ -38,9 +39,11 @@ from AccessControl.SecurityManagement import newSecurityManager
    from DateTime import DateTime
    from Testing import ZopeTestCase
    from Products.ERP5Type.Utils import bytes2str, str2unicode
    from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
    from Products.ERP5Type.tests.utils import DummyTranslationService
    from io import StringIO
    from zExceptions import Unauthorized
    if 1: # BBB
    ......@@ -120,6 +123,11 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
    if 'test_folder' in self.portal.objectIds():
    self.portal.manage_delObjects(['test_folder'])
    self.portal.portal_selections.setSelectionFor('test_selection', None)
    person_module = self.portal.person_module
    person_id_list = list(person_module.objectIds())
    if person_id_list:
    person_module.manage_delObjects(ids=person_id_list)
    self.portal.portal_caches.clearCache()
    self.tic()
    def test_01_ERP5Site_createModule(self, quiet=quiet, run=run_all_test):
    ......@@ -800,3 +808,61 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
    len(self.portal.z_get_deleted_path_list(timestamp=DateTime() - 1)),
    len(person_list) - 5,
    )
    def test_ERP5Site_resynchroniseCatalogSince(self):
    person = self.portal.person_module.newContent(
    portal_type='Person',
    title='test1',
    )
    person2 = self.portal.person_module.newContent(
    portal_type='Person',
    title='test2',
    )
    self.tic()
    query = self.portal.erp5_sql_connection().query
    query('UPDATE catalog SET title="test1bis" WHERE uid=%s' % person.getUid())
    self.assertEqual(
    0,
    len(self.portal.portal_catalog(portal_type='Person', title='test1')),
    )
    # simulate a document being deleted and then the database being
    # truncated before that deletion.
    self.portal.portal_catalog.beforeUncatalogObject(
    uid=person2.getUid(),
    path=person2.getPath(),
    )
    self.assertEqual(
    0,
    len(self.portal.portal_catalog(uid=person2.getUid())),
    )
    response = self.publish(
    '/%s/ERP5Site_resynchroniseCatalogSince?from_date=%s' % (
    self.portal.getId(), DateTime() - 1
    ),
    self.auth,
    )
    self.assertEqual(response.getStatus(), 200)
    io_ = StringIO(str2unicode(bytes2str(response.getBody())))
    response_dict = {x['path']:x for x in csv.DictReader(io_)}
    person_row = response_dict[person.getPath()]
    self.assertEqual(person_row['status'], 'present')
    self.assertEqual(person_row['catalog title'], 'test1bis')
    self.assertEqual(person_row['zodb title'], 'test1')
    person2_row = response_dict[person2.getPath()]
    self.assertEqual(person2_row['status'], 'present')
    self.assertEqual(person2_row['catalog title'], '')
    self.assertEqual(person2_row['zodb title'], 'test2')
    self.portal.ERP5Site_resynchroniseCatalogSince(
    RESPONSE=self.portal.REQUEST.RESPONSE,
    from_date=DateTime() - 1,
    dry=False,
    )
    self.tic()
    self.assertEqual(
    1,
    len(self.portal.portal_catalog(portal_type='Person', title='test1')),
    )
    self.assertEqual(
    1,
    len(self.portal.portal_catalog(uid=person2.getUid())),
    )
    product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_resynchroniseCatalogSince.py 0 → 100644
    View file @ fa7ac3a5
    # Guards: role: Manager, as it is meaningless (and potentially very expensive) to use this script when one cannot typically see all documents.
    import csv
    import six
    from collections import OrderedDict
    from io import BytesIO, StringIO
    from erp5.component.module.Log import log
    if not isinstance(from_date, DateTime):
    from_date = DateTime(from_date)
    assert from_date < DateTime(), from_date
    def always(document_value):
    return True
    def if_has_workflow(document_value):
    return hasattr(document_value, 'workflow_history')
    def unity(value):
    return value
    def strftime(value):
    if value is None:
    return None
    value = value.toZone('UTC')
    return DateTime(
    value.year(),
    value.month(),
    value.day(),
    value.hour(),
    value.minute(),
    int(value.second()),
    'UTC'
    )
    select_dict = OrderedDict((
    # column getter only if getter present ? convert zodb test zodb
    ('uid', ('getUid', False, unity, always)),
    ('title', ('getTitle', False, unity, always)),
    ('portal_type', ('getPortalType', False, unity, always)),
    ('creation_date', ('getCreationDate', False, strftime, always)),
    ('modification_date', ('getModificationDate', False, strftime, if_has_workflow)),
    ('validation_state', ('getValidationState', True, unity, always)),
    ('simulation_state', ('getSimulationState', True, unity, always)),
    ))
    column_list = select_dict.keys()
    portal = context.getPortalObject()
    traverse = portal.restrictedTraverse
    portal_catalog = portal.portal_catalog
    # XXX: abusing CMFActivity's privilege elevation: restricted python is not allowed to call unindexObject
    unindexObject = portal_catalog.activate(activity='SQLQueue',
    group_method_id='portal_catalog/uncatalogObjectList',
    ).unindexObject
    if six.PY2:
    io_ = BytesIO()
    else:
    io_ = StringIO()
    csv_writer = csv.writer(io_)
    row_list = portal_catalog(
    select_list=[e for e in column_list if e != 'uid'],
    indexation_timestamp={
    'query': from_date,
    'range': '>=',
    },
    ).dictionaries()
    row_list.extend(
    portal.z_get_deleted_path_list(timestamp=from_date).dictionaries()
    )
    log('Processing %i rows...' % len(row_list))
    column_title_list = ['status', 'has difference', 'path']
    for column in column_list:
    column_title_list.append('catalog ' + column)
    column_title_list.append('zodb ' + column)
    csv_writer.writerow(column_title_list)
    row_count = len(row_list)
    for i, row in enumerate(row_list):
    zodb_property_dict = {}
    has_difference = False
    try:
    document_value = traverse(row['path'])
    __traceback_info__ = (row['path'], document_value)
    except KeyError:
    status = 'missing'
    has_difference = True
    unindexObject(uid=row['uid'])
    else:
    status = 'present'
    if document_value.getUid() != row['uid']:
    unindexObject(uid=row['uid'])
    for column_name, (getter_name, may_be_missing, zodb_filter, document_filter) in select_dict.items():
    if (
    (not may_be_missing or hasattr(document_value, getter_name)) and
    document_filter(document_value)
    ):
    value_from_document = zodb_filter(getattr(document_value, getter_name)())
    zodb_property_dict[column_name] = value_from_document
    has_difference |= value_from_document != row.get(column_name)
    # Reindex even if no difference was found
    document_value.reindexObject()
    output_value_list = [
    status,
    int(has_difference), # integers are easier to manage than "True" or "False" in libreoffice
    row['path'],
    ]
    output_value_list_append = output_value_list.append
    for column in column_list:
    output_value_list_append(row.get(column))
    output_value_list_append(zodb_property_dict.get(column))
    csv_writer.writerow(output_value_list)
    if i % 1000 == 0:
    log('processed %i/%i lines' % (i, row_count))
    io_.seek(0)
    result = io_.getvalue()
    if dry:
    result += 'dry run'
    if six.PY3:
    result = result.encode()
    RESPONSE.write(result)
    raise Exception('dry run')
    log(result)
    return result
    product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_resynchroniseCatalogSince.xml 0 → 100644
    View file @ fa7ac3a5
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="_reconstructor" module="copy_reg"/>
    </klass>
    <tuple>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    <global name="object" module="__builtin__"/>
    <none/>
    </tuple>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string>RESPONSE, from_date, dry=1</string> </value>
    </item>
    <item>
    <key> <string>guard</string> </key>
    <value>
    <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
    </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>ERP5Site_resynchroniseCatalogSince</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    <record id="2" aka="AAAAAAAAAAI=">
    <pickle>
    <global name="Guard" module="Products.DCWorkflow.Guard"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>roles</string> </key>
    <value>
    <tuple>
    <string>Manager</string>
    </tuple>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7