From 3b781bc8706ef5f0a1cee21b4f2c9134cd577462 Mon Sep 17 00:00:00 2001 From: Vincent Pelletier <vincent@nexedi.com> Date: Thu, 8 Feb 2018 14:44:31 +0900 Subject: [PATCH] ERP5Catalog.test: Move indexation-content-checking tests to a separate file To not receive catalog already tainted by clearing & reindexing. In turn, this allows testing catalog reference content for equality: catalog must be exactly the same before/after an ERP5Site_reindexAll, and before/after a hot-reindex. So issubset is not an acceptable method. Instead, use assertItemsEqual, and raise max diff size. Also, tidy code a bit: - Avoid clearing catalog only to restrict result set. - Improve coding style. - Simplify code (single-use variables...). - Call assert methods producing more helpful error messages on failure. --- product/ERP5Catalog/tests/testERP5Catalog.py | 441 +++++------------- .../tests/testVanillaERP5Catalog.py | 266 +++++++++++ 2 files changed, 376 insertions(+), 331 deletions(-) create mode 100644 product/ERP5Catalog/tests/testVanillaERP5Catalog.py diff --git a/product/ERP5Catalog/tests/testERP5Catalog.py b/product/ERP5Catalog/tests/testERP5Catalog.py index ed4ce88d2b..0a71fe096d 100644 --- a/product/ERP5Catalog/tests/testERP5Catalog.py +++ b/product/ERP5Catalog/tests/testERP5Catalog.py @@ -27,28 +27,20 @@ # ############################################################################## -import unittest +from random import randint import sys +import unittest from unittest import expectedFailure - -from Testing import ZopeTestCase -from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from AccessControl import getSecurityManager from AccessControl.SecurityManagement import newSecurityManager -from zLOG import LOG from DateTime import DateTime -from Products.ERP5Type.tests.utils import LogInterceptor -from Products.ERP5Type.tests.utils import createZODBPythonScript, todo_erp5, \ - getExtraSqlConnectionStringList -from Products.ZSQLCatalog.ZSQLCatalog import HOT_REINDEXING_FINISHED_STATE,\ - HOT_REINDEXING_RECORDING_STATE, HOT_REINDEXING_DOUBLE_INDEXING_STATE -from Products.CMFActivity.Errors import ActivityFlushError +from OFS.ObjectManager import ObjectManager +from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase +from Products.ERP5Type.tests.utils import LogInterceptor, createZODBPythonScript, todo_erp5, getExtraSqlConnectionStringList from Products.PageTemplates.Expressions import getEngine from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery, SimpleQuery - - -from OFS.ObjectManager import ObjectManager -from random import randint +from Testing import ZopeTestCase +from zLOG import LOG class IndexableDocument(ObjectManager): @@ -149,11 +141,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): _, row_list = sql_connection().query(sql, max_rows=0) return [x for x, in row_list] - def getSQLPathListWithRolesAndUsers(self, connection_id): - sql = 'select distinct(path) from catalog, roles_and_users\ - where catalog.security_uid=roles_and_users.uid' - return self.getSQLPathList(connection_id, sql) - def checkRelativeUrlInSQLPathList(self,url_list,connection_id=None): path_list = self.getSQLPathList(connection_id=connection_id) portal_id = self.getPortalId() @@ -268,50 +255,71 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): def test_10_OrderedSearchFolder(self): person_module = self.getPersonModule() - - # Clear catalog - portal_catalog = self.getCatalogTool() - portal_catalog.manage_catalogClear() - - person = person_module.newContent(id='a',portal_type='Person',title='a',description='z') - self.tic() - person = person_module.newContent(id='b',portal_type='Person',title='a',description='y') - self.tic() - person = person_module.newContent(id='c',portal_type='Person',title='a',description='x') - self.tic() - folder_object_list = [x.getObject().getId() - for x in person_module.searchFolder(sort_on=[('id','ascending')])] - self.assertEqual(['a','b','c'],folder_object_list) - folder_object_list = [x.getObject().getId() - for x in person_module.searchFolder( - sort_on=[('title','ascending'), ('description','ascending')])] - self.assertEqual(['c','b','a'],folder_object_list) - folder_object_list = [x.getObject().getId() - for x in person_module.searchFolder( - sort_on=[('title','ascending'),('description','descending')])] - self.assertEqual(['a','b','c'],folder_object_list) + person_uid_list = [ + person_module.newContent(id='a', portal_type='Person', title='a', description='z').getUid(), + person_module.newContent(id='b', portal_type='Person', title='a', description='y').getUid(), + person_module.newContent(id='c', portal_type='Person', title='a', description='x').getUid(), + ] + self.tic() + self.assertEqual( + ['a','b','c'], + [ + x.getObject().getId() + for x in person_module.searchFolder( + sort_on=[('id', 'ascending')], + ) + ], + ) + self.assertEqual( + ['c','b','a'], + [ + x.getObject().getId() + for x in person_module.searchFolder( + sort_on=[('title', 'ascending'), ('description', 'ascending')], + ) + ], + ) + self.assertEqual( + ['a','b','c'], + [ + x.getObject().getId() + for x in person_module.searchFolder( + sort_on=[('title', 'ascending'), ('description', 'descending')], + ) + ], + ) def test_11_CastStringAsInt(self): person_module = self.getPersonModule() - - # Clear catalog - portal_catalog = self.getCatalogTool() - portal_catalog.manage_catalogClear() - - person = person_module.newContent(id='a',portal_type='Person',title='1') - self.tic() - person = person_module.newContent(id='b',portal_type='Person',title='2') - self.tic() - person = person_module.newContent(id='c',portal_type='Person',title='12') - self.tic() - folder_object_list = [x.getObject().getTitle() for x in person_module.searchFolder(sort_on=[('title','ascending')])] - self.assertEqual(['1','12','2'],folder_object_list) - folder_object_list = [x.getObject().getTitle() for x in person_module.searchFolder(sort_on=[('title','ascending','int')])] - self.assertEqual(['1','2','12'],folder_object_list) + person_uid_list = [ + person_module.newContent(portal_type='Person', title='1').getUid(), + person_module.newContent(portal_type='Person', title='2').getUid(), + person_module.newContent(portal_type='Person', title='12').getUid(), + ] + self.tic() + self.assertEqual( + ['1', '12', '2'], + [ + x.getObject().getTitle() + for x in person_module.searchFolder( + sort_on=[('title', 'ascending')], + uid=person_uid_list, + ) + ], + ) + self.assertEqual( + ['1', '2', '12'], + [ + x.getObject().getTitle() + for x in person_module.searchFolder( + sort_on=[('title', 'ascending', 'int')], + uid=person_uid_list, + ) + ], + ) def test_12_TransactionalUidBuffer(self): - portal_catalog = self.getCatalogTool() - catalog = portal_catalog.getSQLCatalog() + catalog = self.getCatalogTool().getSQLCatalog() self.assertTrue(catalog is not None) from Products.ZSQLCatalog.SQLCatalog import global_reserved_uid_lock # Clear out the uid buffer. @@ -337,75 +345,35 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): uid_buffer = getUIDBuffer() self.assertTrue(len(uid_buffer) == 0) - def test_13_ERP5Site_reindexAll(self): - portal = self.getPortal() - self.getCategoryTool().newContent(portal_type='Base Category', title="GreatTitle1") - portal.getDefaultModule('Organisation').newContent(portal_type='Organisation', title="GreatTitle2") - self.tic() - original_path_list = self.getSQLPathList() - self.getCatalogTool().manage_catalogClear() - self.assertEqual([], self.getSQLPathList()) - portal.ERP5Site_reindexAll() - self.tic() - # Check if all objects are catalogued as before - self.assertTrue(set(original_path_list).issubset(self.getSQLPathList())) - def test_14_ReindexWithBrokenCategory(self): """Reindexing an object with 1 broken category must not affect other valid categories""" - # Flush message queue self.tic() - # Create some objects - portal = self.portal portal_category = self.getCategoryTool() - group_nexedi_category = portal_category.group\ - .newContent( id = 'nexedi', ) - region_europe_category = portal_category.region\ - .newContent( id = 'europe', ) - module = portal.getDefaultModule('Organisation') - organisation = module.newContent(portal_type='Organisation',) + group_nexedi_category = portal_category.group.newContent(id='nexedi') + region_europe_category = portal_category.region.newContent(id='europe') + organisation = self.portal.getDefaultModule('Organisation').newContent(portal_type='Organisation') organisation.setGroup('nexedi') self.assertEqual(organisation.getGroupValue(), group_nexedi_category) organisation.setRegion('europe') self.assertEqual(organisation.getRegionValue(), region_europe_category) organisation.setRole('not_exists') self.assertEqual(organisation.getRoleValue(), None) - # Flush message queue self.tic() - # Clear catalog - portal_catalog = self.getCatalogTool() - portal_catalog.manage_catalogClear() sql_connection = self.getSQLConnection() - - sql = 'SELECT COUNT(*) FROM category '\ - 'WHERE uid=%s and category_strict_membership = 1' %\ - organisation.getUid() - result = sql_connection.manage_test(sql) - message_count = result[0]['COUNT(*)'] - self.assertEqual(0, message_count) - # Commit - self.tic() - # Check catalog - organisation.reindexObject() - # Commit - self.tic() - sql = 'select count(*) from message' - result = sql_connection.manage_test(sql) - message_count = result[0]['COUNT(*)'] - self.assertEqual(0, message_count) # Check region and group categories are catalogued - for base_cat, theorical_count in { - 'region':1, - 'group':1, - 'role':0}.items() : - sql = """SELECT COUNT(*) FROM category - WHERE category.uid=%s and category.category_strict_membership = 1 - AND category.base_category_uid = %s""" % (organisation.getUid(), - portal_category[base_cat].getUid()) - result = sql_connection.manage_test(sql) - cataloged_obj_count = result[0]['COUNT(*)'] - self.assertEqual(theorical_count, cataloged_obj_count, - 'category %s is not cataloged correctly' % base_cat) + for base_cat, theorical_count in ( + ('region', 1), + ('group', 1), + ('role', 0), + ): + self.assertEqual( + theorical_count, + sql_connection.manage_test( + "SELECT COUNT(*) FROM category WHERE category.uid=%s and category.category_strict_membership = 1 AND category.base_category_uid = %s" % (organisation.getUid(), portal_category[base_cat].getUid()) + )[0]['COUNT(*)'], + 'category %s is not cataloged correctly' % base_cat, + ) def test_15_getObject(self,): # portal_catalog.getObject raises a ValueError if UID parameter is a string @@ -1074,179 +1042,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): len(self.getCatalogTool()(portal_type='Organisation', limit=None))) ctool.default_result_limit = old_default_result_limit - def test_48_ERP5Site_hotReindexAll(self): - """ - test the hot reindexing of catalog -> catalog2 - then a hot reindexing detailed catalog2 -> catalog - this test use the variable environment: extra_sql_connection_string_list - """ - portal = self.portal - self.original_connection_id = 'erp5_sql_connection' - self.original_deferred_connection_id = self.new_erp5_deferred_sql_connection - self.new_connection_id = self.new_erp5_sql_connection - self.new_deferred_connection_id = 'erp5_sql_deferred_connection2' - new_connection_string = getExtraSqlConnectionStringList()[0] - - # Skip this test if default connection string is not "test test". - original_connection = getattr(portal, self.original_connection_id) - connection_string = original_connection.connection_string - if (connection_string == new_connection_string): - message = 'SKIPPED: default connection string is the same as the one for hot-reindex catalog' - ZopeTestCase._print(message) - LOG('Testing... ',0,message) - - portal_category = self.getCategoryTool() - portal_activities = self.getActivityTool() - self.base_category = portal_category.newContent(portal_type='Base Category', - title="GreatTitle1") - module = portal.getDefaultModule('Organisation') - self.organisation = module.newContent(portal_type='Organisation', - title="GreatTitle2") - # Flush message queue - self.tic() - addSQLConnection = portal.manage_addProduct['ZMySQLDA'] \ - .manage_addZMySQLConnection - # Create new connectors - addSQLConnection(self.new_connection_id,'', new_connection_string) - new_connection = portal[self.new_connection_id] - new_connection.manage_open_connection() - addSQLConnection(self.new_deferred_connection_id,'', new_connection_string) - new_connection = portal[self.new_deferred_connection_id] - new_connection.manage_open_connection() - # the transactionless connector must not be change because this one - # create the portal_ids otherwise it create of conflicts with uid - # objects - - # Create new catalog - portal_catalog = self.getCatalogTool() - self.original_catalog_id = 'erp5_mysql_innodb' - self.new_catalog_id = self.original_catalog_id + '2' - cp_data = portal_catalog.manage_copyObjects(ids=('erp5_mysql_innodb',)) - new_id = portal_catalog.manage_pasteObjects(cp_data)[0]['new_id'] - portal_catalog.manage_renameObject(id=new_id, new_id=self.new_catalog_id) - - # Parse all methods in the new catalog in order to change the connector - new_catalog = portal_catalog[self.new_catalog_id] - source_sql_connection_id_list=list((self.original_connection_id, - self.original_deferred_connection_id)) - destination_sql_connection_id_list=list((self.new_connection_id, - self.new_deferred_connection_id)) - #launch the full hot reindexing - portal_catalog.manage_hotReindexAll(source_sql_catalog_id=self.original_catalog_id, - destination_sql_catalog_id=self.new_catalog_id, - source_sql_connection_id_list=source_sql_connection_id_list, - destination_sql_connection_id_list=destination_sql_connection_id_list, - update_destination_sql_catalog=True) - - # Flush message queue - self.tic() - original_path_list = self.getSQLPathList(self.original_connection_id) - new_path_list = self.getSQLPathList(self.new_connection_id) - self.assertTrue( - set(original_path_list).issubset(new_path_list), - set(original_path_list).difference(new_path_list), - ) - self.organisation2 = module.newContent(portal_type='Organisation', - title="GreatTitle2") - first_deleted_url = self.organisation2.getRelativeUrl() - self.tic() - path_list = [self.organisation.getRelativeUrl()] - self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.original_connection_id) - self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_connection_id) - path_list = [first_deleted_url] - self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=self.original_connection_id) - self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_connection_id) - - # Make sure some zsql method use the right connection_id - zsql_method = portal.portal_skins.erp5_core.Resource_zGetInventoryList - self.assertEqual(getattr(zsql_method,'connection_id'),self.new_connection_id) - - self.assertEqual(portal_catalog.getHotReindexingState(), - HOT_REINDEXING_FINISHED_STATE) - - # Do a hot reindex in the reverse way, but this time a more - # complicated hot reindex - portal_catalog.manage_hotReindexAll( - source_sql_catalog_id=self.new_catalog_id, - destination_sql_catalog_id=self.original_catalog_id, - source_sql_connection_id_list=destination_sql_connection_id_list, - destination_sql_connection_id_list=source_sql_connection_id_list, - update_destination_sql_catalog=True) - self.commit() - self.assertEqual(portal_catalog.getHotReindexingState(), - HOT_REINDEXING_RECORDING_STATE) - self.organisation3 = module.newContent(portal_type='Organisation', - title="GreatTitle2") - # Try something more complicated, create new object, reindex it - # and then delete it - self.deleted_organisation = module.newContent(portal_type='Organisation', - title="GreatTitle2") - self.deleted_organisation.immediateReindexObject() - self.commit() - deleted_url = self.deleted_organisation.getRelativeUrl() - module.manage_delObjects(ids=[self.deleted_organisation.getId()]) - self.commit() - query = self.portal.cmf_activity_sql_connection().query - query( - 'update message set processing_node=-4 where method_id in ' - '("playBackRecordedObjectList", "_finishHotReindexing")', - ) - hasNoProcessableMessage = lambda message_list: all( - x.processing_node == -4 - for x in message_list - ) - self.tic(stop_condition=hasNoProcessableMessage) - self.assertEqual(portal_catalog.getHotReindexingState(), - HOT_REINDEXING_DOUBLE_INDEXING_STATE) - # try to delete objects in double indexing state - module.manage_delObjects(ids=[self.organisation2.getId()]) - self.commit() - query( - 'update message set processing_node=-1 where ' - 'method_id="playBackRecordedObjectList"', - ) - self.tic(stop_condition=hasNoProcessableMessage) - self.assertEqual(portal_catalog.getHotReindexingState(), - HOT_REINDEXING_DOUBLE_INDEXING_STATE) - # Now we have started an double indexing - self.next_deleted_organisation = module.newContent(portal_type='Organisation', - title="GreatTitle2",id='toto') - next_deleted_url = self.next_deleted_organisation.getRelativeUrl() - self.tic(stop_condition=hasNoProcessableMessage) - path_list=[next_deleted_url] - self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.new_connection_id) - self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.original_connection_id) - module.manage_delObjects(ids=[self.next_deleted_organisation.getId()]) - #Create object during the double indexing to check the security object - #after the hot reindexing - self.organisation4 = module.newContent(portal_type='Organisation', - title="GreatTitle2") - self.commit() - query( - 'update message set processing_node=-1 where ' - 'method_id="_finishHotReindexing"', - ) - self.tic() - self.assertEqual(portal_catalog.getHotReindexingState(), - HOT_REINDEXING_FINISHED_STATE) - # Check Security UID object exist in roles and users - # compare the number object in the catalog - count_catalog = len(self.getSQLPathList(self.original_connection_id)) - count_restricted_catalog = len(self.getSQLPathListWithRolesAndUsers(\ - self.original_connection_id)) - self.assertEqual(count_catalog, count_restricted_catalog) - - path_list = [self.organisation3.getRelativeUrl()] - self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.new_connection_id) - self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.original_connection_id) - path_list = [first_deleted_url,deleted_url,next_deleted_url] - self.checkRelativeUrlNotInSQLPathList(path_list,connection_id=self.new_connection_id) - self.checkRelativeUrlNotInSQLPathList(path_list,connection_id=self.original_connection_id) - # Make sure module are there - path_list = [module.getRelativeUrl()] - self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_connection_id) - self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.original_connection_id) - def test_48bis_ERP5Site_hotReindexAllCheckCachedValues(self): """ test the hot reindexing of catalog -> catalog2 @@ -1400,42 +1195,20 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): @todo_erp5 def test_49_IndexInOrderedSearchFolder(self): - person_module = self.getPersonModule() - - # Clear catalog - portal_catalog = self.getCatalogTool() - portal_catalog.manage_catalogClear() - catalog = portal_catalog.objectValues()[0] - - person = person_module.newContent(id='a',portal_type='Person',title='a',description='z') - self.tic() - person = person_module.newContent(id='b',portal_type='Person',title='a',description='y') + searchFolder = self.getPersonModule().searchFolder + catalog = self.getCatalogTool().objectValues()[0] self.tic() - person = person_module.newContent(id='c',portal_type='Person',title='a',description='x') - self.tic() - index_columns = getattr(catalog, 'sql_catalog_index_on_order_keys', None) - self.assertNotEqual(index_columns, None) - self.assertEqual(len(index_columns), 0) + self.assertEqual(catalog.sql_catalog_index_on_order_keys, ()) # Check catalog don't tell to use index if nothing defined - sql = person_module.searchFolder(src__=1) - self.assertTrue('use index' not in sql) - sql = person_module.searchFolder(src__=1, sort_on=[('id','ascending')]) - self.assertTrue('use index' not in sql) - sql = person_module.searchFolder(src__=1, sort_on=[('title','ascending')]) - self.assertTrue('use index' not in sql) + self.assertNotIn('use index', searchFolder(src__=1)) + self.assertNotIn('use index', searchFolder(src__=1, sort_on=[('id','ascending')])) + self.assertNotIn('use index', searchFolder(src__=1, sort_on=[('title','ascending')])) # Defined that catalog must tell to use index when order by catalog.title - index_columns = ('catalog.title',) - setattr(catalog, 'sql_catalog_index_on_order_keys', index_columns) - index_columns = getattr(catalog, 'sql_catalog_index_on_order_keys', None) - self.assertNotEqual(index_columns, None) - self.assertEqual(len(index_columns), 1) + catalog.sql_catalog_index_on_order_keys = ('catalog.title', ) # Check catalog tell to use index only when ordering by catalog.title - sql = person_module.searchFolder(src__=1) - self.assertTrue('use index' not in sql) - sql = person_module.searchFolder(src__=1, sort_on=[('id','ascending')]) - self.assertTrue('use index' not in sql) - sql = person_module.searchFolder(src__=1, sort_on=[('title','ascending')]) - self.assertTrue('use index' in sql) + self.assertNotIn('use index', searchFolder(src__=1)) + self.assertNotIn('use index', searchFolder(src__=1, sort_on=[('id','ascending')])) + self.assertIn('use index', searchFolder(src__=1, sort_on=[('title','ascending')])) def test_50_LocalRolesArgument(self): """test local_roles= argument @@ -1662,17 +1435,25 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): # Add a script to create uid list catalog = self.getCatalogTool().getSQLCatalog() script_id = 'z0_zCreateUid' - script_content = "context.getPortalObject().portal_ids.generateNewIdList(id_generator='uid',\ - id_group='text_uid')" - script = createZODBPythonScript(catalog, script_id, - '*args,**kw', script_content) - sql_clear_catalog = list(catalog.sql_clear_catalog) - sql_clear_catalog.append(script_id) - sql_clear_catalog.sort() - catalog.sql_clear_catalog = tuple(sql_clear_catalog) + script = createZODBPythonScript( + catalog, + script_id, + '*args,**kw', + "context.getPortalObject().portal_ids.generateNewIdList(id_generator='uid', id_group='text_uid')", + ) + sql_clear_catalog_orig = catalog.sql_clear_catalog + catalog.sql_clear_catalog = tuple(sorted(sql_clear_catalog_orig + (script_id, ))) # launch the sql_clear_catalog with the script after the drop tables and # before the recreate tables of catalog - catalog.manage_catalogClear() + try: + self.commit() + catalog.manage_catalogClear() + finally: + self.abort() + catalog.sql_clear_catalog = sql_clear_catalog_orig + self.commit() + self.portal.ERP5Site_reindexAll(clear_catalog=True) + self.tic() def test_SearchOnOwner(self): # owner= can be used a search key in the catalog to have all documents for @@ -2042,9 +1823,7 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): uf._doAddUser('foo', 'foo', ['Member', ], []) uf._doAddUser('ERP5TypeTestCase', 'ERP5TypeTestCase', ['Member', ], []) self.commit() - portal_catalog = self.getCatalogTool() - portal_catalog.manage_catalogClear() - self.getPortal().ERP5Site_reindexAll() + self.getPortal().ERP5Site_reindexAll(clear_catalog=True) self.tic() # Person stuff diff --git a/product/ERP5Catalog/tests/testVanillaERP5Catalog.py b/product/ERP5Catalog/tests/testVanillaERP5Catalog.py new file mode 100644 index 0000000000..4847ec8c0f --- /dev/null +++ b/product/ERP5Catalog/tests/testVanillaERP5Catalog.py @@ -0,0 +1,266 @@ +############################################################################## +# +# Copyright (c) 2018 Nexedi SARL and Contributors. All Rights Reserved. +# Vincent Pelletier <vincent@nexedi.com> +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import unittest +from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase +from Products.ERP5Type.tests.utils import LogInterceptor, getExtraSqlConnectionStringList +from Products.ZSQLCatalog.ZSQLCatalog import HOT_REINDEXING_FINISHED_STATE, HOT_REINDEXING_RECORDING_STATE, HOT_REINDEXING_DOUBLE_INDEXING_STATE +from Testing import ZopeTestCase +from zLOG import LOG + +class TestVanillaERP5Catalog(ERP5TypeTestCase, LogInterceptor): + """ + Tests for ERP5 Catalog where a pristine freshly-created-site catalog is + needed. Clearing catalog is only allowed when comparing before/after for + strict item equality. + """ + + def getTitle(self): + return "VanillaERP5Catalog" + + def getBusinessTemplateList(self): + return ('erp5_full_text_mroonga_catalog', 'erp5_base') + + # Different variables used for this test + username = 'seb' + new_erp5_sql_connection = 'erp5_sql_connection2' + new_erp5_deferred_sql_connection = 'erp5_sql_deferred_connection2' + original_catalog_id = 'erp5_mysql_innodb' + new_catalog_id = 'erp5_mysql_innodb2' + + def afterSetUp(self): + portal = self.portal + portal.acl_users._doAddUser(self.username, '', ['Manager'], []) + self.loginByUserName(self.username) + self.tic() + + def beforeTearDown(self): + # restore default_catalog + portal = self.portal + portal.portal_catalog._setDefaultSqlCatalogId(self.original_catalog_id) + portal.portal_catalog.hot_reindexing_state = None + module = self.getOrganisationModule() + module.manage_delObjects(list(module.objectIds())) + module.reindexObject() + # Remove copied sql_connector and catalog + if self.new_erp5_sql_connection in portal.objectIds(): + portal.manage_delObjects([self.new_erp5_sql_connection]) + if self.new_erp5_deferred_sql_connection in portal.objectIds(): + portal.manage_delObjects([self.new_erp5_deferred_sql_connection]) + if self.new_catalog_id in portal.portal_catalog.objectIds(): + portal.portal_catalog.manage_delObjects([self.new_catalog_id]) + self.tic() + + def getSQLPathList(self,connection_id=None, sql=None): + """ + Give the full list of path in the catalog + """ + if connection_id is None: + sql_connection = self.getSQLConnection() + else: + sql_connection = getattr(self.getPortal(), connection_id) + if sql is None: + sql = 'select distinct(path) from catalog' + _, row_list = sql_connection().query(sql, max_rows=0) + return [x for x, in row_list] + + def getSQLPathListWithRolesAndUsers(self, connection_id): + sql = 'select distinct(path) from catalog, roles_and_users\ + where catalog.security_uid=roles_and_users.uid' + return self.getSQLPathList(connection_id, sql) + + def checkRelativeUrlInSQLPathList(self,url_list,connection_id=None): + path_list = self.getSQLPathList(connection_id=connection_id) + portal_id = self.getPortalId() + for url in url_list: + path = '/' + portal_id + '/' + url + self.assertTrue(path in path_list) + LOG('checkRelativeUrlInSQLPathList found path:',0,path) + + def checkRelativeUrlNotInSQLPathList(self,url_list,connection_id=None): + path_list = self.getSQLPathList(connection_id=connection_id) + portal_id = self.getPortalId() + for url in url_list: + path = '/' + portal_id + '/' + url + self.assertTrue(path not in path_list) + LOG('checkRelativeUrlInSQLPathList not found path:',0,path) + + def test_1_ERP5Site_reindexAll(self): + portal = self.getPortal() + portal.portal_categories.newContent(portal_type='Base Category', title="GreatTitle1") + portal.organisation_module.newContent(portal_type='Organisation', title="GreatTitle2") + self.tic() + original_path_list = self.getSQLPathList() + self.getCatalogTool().manage_catalogClear() + self.assertEqual([], self.getSQLPathList()) + portal.ERP5Site_reindexAll() + self.tic() + # Check if all objects are catalogued as before + self.maxDiff = None + self.assertItemsEqual(original_path_list, self.getSQLPathList()) + + def test_2_ERP5Site_hotReindexAll(self): + """ + test the hot reindexing of catalog -> catalog2 + then a hot reindexing detailed catalog2 -> catalog + this test use the variable environment: extra_sql_connection_string_list + """ + portal = self.portal + original_connection_id = 'erp5_sql_connection' + extra_connection_string_list = getExtraSqlConnectionStringList() + if not extra_connection_string_list or extra_connection_string_list[0] == getattr(portal, original_connection_id).connection_string: + self.skipTest('default connection string is the same as the one for hot-reindex catalog') + new_connection_string = extra_connection_string_list[0] + new_deferred_connection_id = 'erp5_sql_deferred_connection2' + module = portal.organisation_module + organisation = module.newContent(portal_type='Organisation', title="GreatTitle2") + self.tic() + addSQLConnection = portal.manage_addProduct['ZMySQLDA'].manage_addZMySQLConnection + # Create new connectors + addSQLConnection(self.new_erp5_sql_connection, '', new_connection_string) + portal[self.new_erp5_sql_connection].manage_open_connection() + addSQLConnection(new_deferred_connection_id, '', new_connection_string) + portal[new_deferred_connection_id].manage_open_connection() + # Note: transactionless connector must not be changed because this one + # create the portal_ids otherwise it create of conflicts with uid + # objects. + # Create new catalog + portal_catalog = portal.portal_catalog + portal_catalog.manage_renameObject( + id=portal_catalog.manage_pasteObjects( + portal_catalog.manage_copyObjects(ids=(self.original_catalog_id, )), + )[0]['new_id'], + new_id=self.new_catalog_id, + ) + source_sql_connection_id_list = [original_connection_id, self.new_erp5_deferred_sql_connection] + destination_sql_connection_id_list = [self.new_erp5_sql_connection, new_deferred_connection_id] + portal_catalog.manage_hotReindexAll( + source_sql_catalog_id=self.original_catalog_id, + destination_sql_catalog_id=self.new_catalog_id, + source_sql_connection_id_list=source_sql_connection_id_list, + destination_sql_connection_id_list=destination_sql_connection_id_list, + update_destination_sql_catalog=True, + ) + self.tic() + original_path_list = self.getSQLPathList(original_connection_id) + new_path_list = self.getSQLPathList(self.new_erp5_sql_connection) + self.maxDiff = None + self.assertItemsEqual(original_path_list, new_path_list) + organisation2 = module.newContent(portal_type='Organisation', title="GreatTitle2") + first_deleted_url = organisation2.getRelativeUrl() + self.tic() + path_list = [organisation.getRelativeUrl()] + self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id) + self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection) + path_list = [first_deleted_url] + self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=original_connection_id) + self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection) + + self.assertEqual(portal.portal_skins.erp5_core.Resource_zGetInventoryList.connection_id, self.new_erp5_sql_connection) + self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_FINISHED_STATE) + + # Do a hot reindex in the reverse way, but this time a more + # complicated hot reindex + portal_catalog.manage_hotReindexAll( + source_sql_catalog_id=self.new_catalog_id, + destination_sql_catalog_id=self.original_catalog_id, + source_sql_connection_id_list=destination_sql_connection_id_list, + destination_sql_connection_id_list=source_sql_connection_id_list, + update_destination_sql_catalog=True, + ) + self.commit() + self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_RECORDING_STATE) + organisation3 = module.newContent(portal_type='Organisation', title="GreatTitle2") + # Try something more complicated, create new object, reindex it + # and then delete it + deleted_organisation = module.newContent(portal_type='Organisation', title="GreatTitle2") + deleted_organisation.immediateReindexObject() + self.commit() + deleted_url = deleted_organisation.getRelativeUrl() + module.manage_delObjects(ids=[deleted_organisation.getId()]) + self.commit() + query = self.portal.cmf_activity_sql_connection().query + query( + 'update message set processing_node=-4 where method_id in ' + '("playBackRecordedObjectList", "_finishHotReindexing")', + ) + hasNoProcessableMessage = lambda message_list: all( + x.processing_node == -4 + for x in message_list + ) + self.tic(stop_condition=hasNoProcessableMessage) + self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_DOUBLE_INDEXING_STATE) + # try to delete objects in double indexing state + module.manage_delObjects(ids=[organisation2.getId()]) + self.commit() + query( + 'update message set processing_node=-1 where ' + 'method_id="playBackRecordedObjectList"', + ) + self.tic(stop_condition=hasNoProcessableMessage) + self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_DOUBLE_INDEXING_STATE) + # Now we have started an double indexing + next_deleted_organisation = module.newContent(portal_type='Organisation', title="GreatTitle2",id='toto') + next_deleted_url = next_deleted_organisation.getRelativeUrl() + self.tic(stop_condition=hasNoProcessableMessage) + path_list=[next_deleted_url] + self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection) + self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id) + module.manage_delObjects(ids=[next_deleted_organisation.getId()]) + # Create object during the double indexing to check the security object + # after the hot reindexing + module.newContent(portal_type='Organisation', title="GreatTitle2") + self.commit() + query( + 'update message set processing_node=-1 where ' + 'method_id="_finishHotReindexing"', + ) + self.tic() + self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_FINISHED_STATE) + # Check Security UID object exist in roles and users + # compare the number object in the catalog + self.assertItemsEqual( + self.getSQLPathList(original_connection_id), + self.getSQLPathListWithRolesAndUsers(original_connection_id), + ) + + path_list = [organisation3.getRelativeUrl()] + self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection) + self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id) + path_list = [first_deleted_url, deleted_url,next_deleted_url] + self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection) + self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=original_connection_id) + # Make sure module are there + path_list = [module.getRelativeUrl()] + self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection) + self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id) + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestVanillaERP5Catalog)) + return suite -- 2.30.9