Skip to content

erp5
erp5
Commit fa7ac3a5 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki
Browse files
Options

erp5_core: add ERP5Site_resynchroniseCatalogSince script.

parent 70b5d264
Hide whitespace changes
Inline Side-by-side
Showing with 272 additions and 0 deletions
bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5Core.py
View file @ fa7ac3a5
......@@ -28,6 +28,7 @@
##############################################################################
import collections
import csv
import httplib
import urlparse
import base64
......@@ -38,9 +39,11 @@ from AccessControl.SecurityManagement import newSecurityManager
from DateTime import DateTime
from Testing import ZopeTestCase
from Products.ERP5Type.Utils import bytes2str, str2unicode
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import DummyTranslationService
from io import StringIO
from zExceptions import Unauthorized
if 1: # BBB
......@@ -120,6 +123,11 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
if 'test_folder' in self.portal.objectIds():
self.portal.manage_delObjects(['test_folder'])
self.portal.portal_selections.setSelectionFor('test_selection', None)
person_module = self.portal.person_module
person_id_list = list(person_module.objectIds())
if person_id_list:
person_module.manage_delObjects(ids=person_id_list)
self.portal.portal_caches.clearCache()
self.tic()
def test_01_ERP5Site_createModule(self, quiet=quiet, run=run_all_test):
......@@ -800,3 +808,61 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
len(self.portal.z_get_deleted_path_list(timestamp=DateTime() - 1)),
len(person_list) - 5,
)
def test_ERP5Site_resynchroniseCatalogSince(self):
person = self.portal.person_module.newContent(
portal_type='Person',
title='test1',
)
person2 = self.portal.person_module.newContent(
portal_type='Person',
title='test2',
)
self.tic()
query = self.portal.erp5_sql_connection().query
query('UPDATE catalog SET title="test1bis" WHERE uid=%s' % person.getUid())
self.assertEqual(
0,
len(self.portal.portal_catalog(portal_type='Person', title='test1')),
)
# simulate a document being deleted and then the database being
# truncated before that deletion.
self.portal.portal_catalog.beforeUncatalogObject(
uid=person2.getUid(),
path=person2.getPath(),
)
self.assertEqual(
0,
len(self.portal.portal_catalog(uid=person2.getUid())),
)
response = self.publish(
'/%s/ERP5Site_resynchroniseCatalogSince?from_date=%s' % (
self.portal.getId(), DateTime() - 1
),
self.auth,
)
self.assertEqual(response.getStatus(), 200)
io_ = StringIO(str2unicode(bytes2str(response.getBody())))
response_dict = {x['path']:x for x in csv.DictReader(io_)}
person_row = response_dict[person.getPath()]
self.assertEqual(person_row['status'], 'present')
self.assertEqual(person_row['catalog title'], 'test1bis')
self.assertEqual(person_row['zodb title'], 'test1')
person2_row = response_dict[person2.getPath()]
self.assertEqual(person2_row['status'], 'present')
self.assertEqual(person2_row['catalog title'], '')
self.assertEqual(person2_row['zodb title'], 'test2')
self.portal.ERP5Site_resynchroniseCatalogSince(
RESPONSE=self.portal.REQUEST.RESPONSE,
from_date=DateTime() - 1,
dry=False,
)
self.tic()
self.assertEqual(
1,
len(self.portal.portal_catalog(portal_type='Person', title='test1')),
)
self.assertEqual(
1,
len(self.portal.portal_catalog(uid=person2.getUid())),
)
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_resynchroniseCatalogSince.py 0 → 100644
View file @ fa7ac3a5
# Guards: role: Manager, as it is meaningless (and potentially very expensive) to use this script when one cannot typically see all documents.
import csv
import six
from collections import OrderedDict
from io import BytesIO, StringIO
from erp5.component.module.Log import log
if not isinstance(from_date, DateTime):
from_date = DateTime(from_date)
assert from_date < DateTime(), from_date
def always(document_value):
return True
def if_has_workflow(document_value):
return hasattr(document_value, 'workflow_history')
def unity(value):
return value
def strftime(value):
if value is None:
return None
value = value.toZone('UTC')
return DateTime(
value.year(),
value.month(),
value.day(),
value.hour(),
value.minute(),
int(value.second()),
'UTC'
)
select_dict = OrderedDict((
# column getter only if getter present ? convert zodb test zodb
('uid', ('getUid', False, unity, always)),
('title', ('getTitle', False, unity, always)),
('portal_type', ('getPortalType', False, unity, always)),
('creation_date', ('getCreationDate', False, strftime, always)),
('modification_date', ('getModificationDate', False, strftime, if_has_workflow)),
('validation_state', ('getValidationState', True, unity, always)),
('simulation_state', ('getSimulationState', True, unity, always)),
))
column_list = select_dict.keys()
portal = context.getPortalObject()
traverse = portal.restrictedTraverse
portal_catalog = portal.portal_catalog
# XXX: abusing CMFActivity's privilege elevation: restricted python is not allowed to call unindexObject
unindexObject = portal_catalog.activate(activity='SQLQueue',
group_method_id='portal_catalog/uncatalogObjectList',
).unindexObject
if six.PY2:
io_ = BytesIO()
else:
io_ = StringIO()
csv_writer = csv.writer(io_)
row_list = portal_catalog(
select_list=[e for e in column_list if e != 'uid'],
indexation_timestamp={
'query': from_date,
'range': '>=',
},
).dictionaries()
row_list.extend(
portal.z_get_deleted_path_list(timestamp=from_date).dictionaries()
)
log('Processing %i rows...' % len(row_list))
column_title_list = ['status', 'has difference', 'path']
for column in column_list:
column_title_list.append('catalog ' + column)
column_title_list.append('zodb ' + column)
csv_writer.writerow(column_title_list)
row_count = len(row_list)
for i, row in enumerate(row_list):
zodb_property_dict = {}
has_difference = False
try:
document_value = traverse(row['path'])
__traceback_info__ = (row['path'], document_value)
except KeyError:
status = 'missing'
has_difference = True
unindexObject(uid=row['uid'])
else:
status = 'present'
if document_value.getUid() != row['uid']:
unindexObject(uid=row['uid'])
for column_name, (getter_name, may_be_missing, zodb_filter, document_filter) in select_dict.items():
if (
(not may_be_missing or hasattr(document_value, getter_name)) and
document_filter(document_value)
):
value_from_document = zodb_filter(getattr(document_value, getter_name)())
zodb_property_dict[column_name] = value_from_document
has_difference |= value_from_document != row.get(column_name)
# Reindex even if no difference was found
document_value.reindexObject()
output_value_list = [
status,
int(has_difference), # integers are easier to manage than "True" or "False" in libreoffice
row['path'],
]
output_value_list_append = output_value_list.append
for column in column_list:
output_value_list_append(row.get(column))
output_value_list_append(zodb_property_dict.get(column))
csv_writer.writerow(output_value_list)
if i % 1000 == 0:
log('processed %i/%i lines' % (i, row_count))
io_.seek(0)
result = io_.getvalue()
if dry:
result += 'dry run'
if six.PY3:
result = result.encode()
RESPONSE.write(result)
raise Exception('dry run')
log(result)
return result
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_resynchroniseCatalogSince.xml 0 → 100644
View file @ fa7ac3a5
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>RESPONSE, from_date, dry=1</string> </value>
</item>
<item>
<key> <string>guard</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_resynchroniseCatalogSince</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Guard" module="Products.DCWorkflow.Guard"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>roles</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7