Skip to content

erp5
erp5
Commit 6a6795b3 authored by Aurel's avatar Aurel Committed by Tristan Cavelier
Browse files
Options

implement fulltext search for title & description 

Conflicts:
	product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/revision
	product/ZSQLCatalog/SearchKey/FullTextKey.py
parent 143b3b40
Hide whitespace changes
Inline Side-by-side
Showing with 487 additions and 36 deletions
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_makeDescriptionFullTextQuery.xml 0 → 100644
View file @ 6a6795b3
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
MySQL\'s full text searchable key implementation.\n
"""\n
from Products.ZSQLCatalog.SQLCatalog import Query\n
\n
query = Query(**{\'catalog_full_text.fulltext_description\': value} )\n
return query\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>value</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_makeDescriptionFullTextQuery</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_makeFullTextQuery.xml
View file @ 6a6795b3
......@@ -51,14 +51,11 @@
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
Default full text searchable key implementation.\n
MySQL\'s full text searchable key implementation.\n
"""\n
from Products.ZSQLCatalog.SQLCatalog import ComplexQuery\n
from Products.ZSQLCatalog.SQLCatalog import Query\n
\n
query = ComplexQuery(Query(title=value),\n
Query(reference=value),\n
operator="OR")\n
query = Query(**{\'full_text.SearchableText\': value})\n
return query\n
</string> </value>
</item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_makeTitleFullTextQuery.xml 0 → 100644
View file @ 6a6795b3
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
MySQL\'s full text searchable key implementation.\n
"""\n
from Products.ZSQLCatalog.SQLCatalog import Query\n
\n
query = Query(**{\'catalog_full_text.fulltext_title\': value} )\n
return query\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>value</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_makeTitleFullTextQuery</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_catalog_fulltext.catalog_keys.xml 0 → 100644
View file @ 6a6795b3
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_catalog_fulltext.xml 0 → 100644
View file @ 6a6795b3
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS catalog_full_text</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_catalog_fulltext.catalog_keys.xml 0 → 100644
View file @ 6a6795b3
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_catalog_fulltext.xml 0 → 100644
View file @ 6a6795b3
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_deferred_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_object_fulltext_list.catalog_keys.xml 0 → 100644
View file @ 6a6795b3
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_object_fulltext_list.xml 0 → 100644
View file @ 6a6795b3
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid\r\n
getTitle\r\n
getDescription</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_object_fulltext_list</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
REPLACE INTO\n
catalog_full_text (`uid`, `fulltext_title`, `fulltext_description`)\n
VALUES\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
(\n
<dtml-sqlvar expr="uid[loop_item]" type="int">, \n
<dtml-sqlvar expr="getTitle[loop_item]" type="string" optional>,\n
<dtml-sqlvar expr="getDescription[loop_item]" type="string" optional>\n
)\n
<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_catalog_fulltext.xml 0 → 100644
View file @ 6a6795b3
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string># Host:\n
# Database: test\n
# Table: \'catalog_full_text\'\n
#\n
CREATE TABLE `catalog_full_text` (\n
`uid` BIGINT UNSIGNED NOT NULL,\n
`fulltext_title` varchar(255) default \'\',\n
`fulltext_description` text,\n
PRIMARY KEY (`uid`),\n
FULLTEXT `title` (`fulltext_title`) COMMENT \'parser "TokenBigramSplitSymbolAlpha"\',\n
FULLTEXT `description` (`fulltext_description`) COMMENT \'parser "TokenBigramSplitSymbolAlpha"\'\n
) ENGINE=mroonga COMMENT=\'engine "innodb"\';\n
</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogResultTableTemplateItem/result_table_list.xml
View file @ 6a6795b3
<key_list>
<key>alarm</key>
<key>catalog</key>
<key>catalog_full_text</key>
<key>category</key>
<key>delivery</key>
<key>item</key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogScriptableKeyTemplateItem/scriptable_key_list.xml
View file @ 6a6795b3
<key_list>
<key>advanced_search_text | SQLCatalog_makeQuickSearchQuery</key>
<key>description | SQLCatalog_makeDescriptionFullTextQuery</key>
<key>full_text | SQLCatalog_makeFullTextQuery</key>
<key>quick_search_text | SQLCatalog_makeQuickSearchQuery</key>
<key>search_text | SQLCatalog_makeSearchTextQuery</key>
<key>title | SQLCatalog_makeTitleFullTextQuery</key>
<key>translated_title | SQLCatalog_makeTranslatedTitleQuery</key>
</key_list>
\ No newline at end of file
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ 6a6795b3
erp5_mysql_innodb/SQLCatalog_catalogTransformation
erp5_mysql_innodb/SQLCatalog_catalogTransformationList
erp5_mysql_innodb/SQLCatalog_makeDescriptionFullTextQuery
erp5_mysql_innodb/SQLCatalog_makeFullTextQuery
erp5_mysql_innodb/SQLCatalog_makeQuickSearchQuery
erp5_mysql_innodb/SQLCatalog_makeSearchTextQuery
erp5_mysql_innodb/SQLCatalog_makeTitleFullTextQuery
erp5_mysql_innodb/SQLCatalog_makeTranslatedTitleQuery
erp5_mysql_innodb/z0_drop_alarm
erp5_mysql_innodb/z0_drop_catalog
erp5_mysql_innodb/z0_drop_catalog_fulltext
erp5_mysql_innodb/z0_drop_category
erp5_mysql_innodb/z0_drop_delivery
erp5_mysql_innodb/z0_drop_item
......@@ -21,6 +24,7 @@ erp5_mysql_innodb/z0_drop_transformation
erp5_mysql_innodb/z0_drop_translation
erp5_mysql_innodb/z0_drop_versioning
erp5_mysql_innodb/z0_uncatalog_alarm
erp5_mysql_innodb/z0_uncatalog_catalog_fulltext
erp5_mysql_innodb/z0_uncatalog_category
erp5_mysql_innodb/z0_uncatalog_item
erp5_mysql_innodb/z0_uncatalog_measure
......@@ -36,6 +40,7 @@ erp5_mysql_innodb/z_catalog_item_list
erp5_mysql_innodb/z_catalog_measure_list
erp5_mysql_innodb/z_catalog_movement_category_list
erp5_mysql_innodb/z_catalog_non_movement_category_list
erp5_mysql_innodb/z_catalog_object_fulltext_list
erp5_mysql_innodb/z_catalog_object_list
erp5_mysql_innodb/z_catalog_paths
erp5_mysql_innodb/z_catalog_predicate_category_list
......@@ -52,6 +57,7 @@ erp5_mysql_innodb/z_clear_reserved
erp5_mysql_innodb/z_count_results
erp5_mysql_innodb/z_create_alarm
erp5_mysql_innodb/z_create_catalog
erp5_mysql_innodb/z_create_catalog_fulltext
erp5_mysql_innodb/z_create_category
erp5_mysql_innodb/z_create_delivery
erp5_mysql_innodb/z_create_item
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_scriptable_key_list
View file @ 6a6795b3
......@@ -2,4 +2,6 @@ quick_search_text | SQLCatalog_makeQuickSearchQuery
advanced_search_text | SQLCatalog_makeQuickSearchQuery
full_text | SQLCatalog_makeFullTextQuery
search_text | SQLCatalog_makeSearchTextQuery
translated_title | SQLCatalog_makeTranslatedTitleQuery
\ No newline at end of file
translated_title | SQLCatalog_makeTranslatedTitleQuery
title | SQLCatalog_makeTitleFullTextQuery
description | SQLCatalog_makeDescriptionFullTextQuery
\ No newline at end of file
product/ZSQLCatalog/SearchKey/FullTextKey.py
View file @ 6a6795b3
......@@ -33,15 +33,12 @@ from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
from Products.ZSQLCatalog.SearchText import parse
from Products.ZSQLCatalog.interfaces.search_key import ISearchKey
from zope.interface.verify import verifyClass
import re
FULLTEXT_BOOLEAN_DETECTOR = re.compile(r'.*((^|\s)[\+\-<>\(\~]|[\*\)](\s|$))')
class FullTextKey(SearchKey):
"""
This SearchKey generates SQL fulltext comparisons.
"""
default_comparison_operator = 'match'
default_comparison_operator = 'boolean_match'
get_operator_from_value = False
def parseSearchText(self, value, is_column):
......@@ -53,32 +50,6 @@ class FullTextKey(SearchKey):
def _renderValueAsSearchText(self, value, operator):
return '(%s)' % (value, )
def _processSearchValue(self, search_value, logical_operator,
comparison_operator):
"""
Special SearchValue processor for FullText queries: if a searched value
from 'match' operator group contains an operator recognised in boolean
mode, make the operator for that value be 'match_boolean'.
"""
operator_value_dict, logical_operator, parsed = \
SearchKey._processSearchValue(self, search_value, logical_operator,
comparison_operator)
new_value_list = []
append = new_value_list.append
for value in operator_value_dict.pop('match', []):
if isinstance(value, basestring) and \
FULLTEXT_BOOLEAN_DETECTOR.match(value) is not None:
operator_value_dict.setdefault('match_boolean', []).append(value)
else:
append(value)
if len(new_value_list):
if 'match_boolean' in operator_value_dict:
# use boolean mode for all expressions
operator_value_dict['match_boolean'].extend(new_value_list)
else:
operator_value_dict['match'] = new_value_list
return operator_value_dict, logical_operator, parsed
def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
"""
Special Query builder for FullText queries: merge all values having the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7