Add auto_extend_select_list argument in buildSQLQuery() and use alias in...

Add auto_extend_select_list argument in buildSQLQuery() and use alias in group_by_expression and order_by_expression.

If True, select_list is automatically extended to have columns used in
group_by_list and order_by_list. It is useful when use
select_expression in inner query and use group_by_expression or
order_by_expression in outer query.

product/ERP5/tests/testI18NSearch.py

@@ -80,10 +80,10 @@ class TestI18NSearch(ERP5TypeTestCase):
     self.assertEqual(result[0].getPath(), self.person1.getPath())
 
     # check sort on fulltext column
-    self.assertFalse('ORDER BY\n  MATCH' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText', 'ascending'),), src__=1))
+    self.assertTrue('ORDER BY\n  `full_text`.`SearchableText` ASC' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText', 'ascending'),), src__=1))
 
     # check sort on fulltext search score
-    self.assertTrue('ORDER BY\n  MATCH' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText__score__', 'ascending'),), src__=1))
+    self.assertTrue('ORDER BY\n  full_text_SearchableText__score__ ASC' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText__score__', 'ascending'),), src__=1))
 
   def test_catalog_full_text_title(self):
     # check if 'é' == 'e' collation works
@@ -108,18 +108,23 @@ class TestI18NSearch(ERP5TypeTestCase):
       self.assertEqual(result[0].getPath(), self.person3.getPath())
 
     # check sort on fulltext column
-    self.assertFalse('ORDER BY\n  MATCH' in self.portal.portal_catalog(**{
+    self.assertFalse('ORDER BY\n  catalog_full_text_title__score__ ASC' in self.portal.portal_catalog(**{
       'catalog_full_text.title':'Faure',
       'sort_on':(('catalog_full_text.title', 'ascending'),),
       'src__':1
       }))
 
     # check sort on fulltext search score
-    self.assertFalse('ORDER BY\n  MATCH' in self.portal.portal_catalog(**{
+    self.assertTrue('ORDER BY\n  catalog_full_text_title__score__' in self.portal.portal_catalog(**{
       'catalog_full_text.title':'Faure',
       'sort_on':(('catalog_full_text.title__score__', 'ascending'),),
       'src__':1
       }))
+    self.assertTrue('ORDER BY\n  catalog_full_text_title__score__' in self.portal.portal_catalog(**{
+      'catalog_full_text.title':'Faure',
+      'sort_on':(('title__score__', 'ascending'),),
+      'src__':1
+      }))
 
   @expectedFailure
   def test_full_text_title(self):
@@ -148,10 +153,10 @@ class TestI18NSearch(ERP5TypeTestCase):
     self.assertTrue('MATCH' in self.portal.portal_catalog(destination_title='Faure', src__=1))
 
     # check sort on fulltext column
-    self.assertFalse('ORDER BY\n  MATCH' in self.portal.portal_catalog(title='Faure', sort_on=(('title', 'ascending'),), src__=1))
+    self.assertTrue('ORDER BY\n  `catalog`.`title` ASC' in self.portal.portal_catalog(title='Faure', sort_on=(('title', 'ascending'),), src__=1))
 
     # check sort on fulltext search score
-    self.assertTrue('ORDER BY\n  MATCH' in self.portal.portal_catalog(title='Faure', sort_on=(('title__score__', 'ascending'),), src__=1))
+    self.assertTrue('ORDER BY\n  catalog_full_text_title__score__' in self.portal.portal_catalog(title='Faure', sort_on=(('title__score__', 'ascending'),), src__=1))
 
 def test_suite():
   suite = unittest.TestSuite()

product/ZSQLCatalog/ColumnMap.py

@@ -453,7 +453,10 @@ class ColumnMap(object):
   def asSQLColumn(self, raw_column, group=DEFAULT_GROUP_ID):
     if self.catalog_table_name is None or raw_column in self.column_ignore_set or \
        '.' in raw_column or '*' in raw_column:
-      result = raw_column
+      if raw_column.endswith('__score__'):
+        result = raw_column.replace('.', '_')
+      else:
+        result = raw_column
     else:
       if raw_column.endswith('__score__'):
         raw_column = raw_column[:-9]
@@ -464,7 +467,10 @@ class ColumnMap(object):
       if group is DEFAULT_GROUP_ID:
         group, column = self.related_key_dict.get(column, (group, raw_column))
       alias = self.table_alias_dict[(group, self.column_map[(group, column)])]
-      result = '`%s`.`%s%s`' % (alias, column, column_suffix)
+      if column_suffix:
+        result = '%s_%s%s' % (alias, column, column_suffix)
+      else:
+        result = '`%s`.`%s`' % (alias, column)
       if function is not None:
         result = '%s(%s)' % (function, result)
     return result

product/ZSQLCatalog/Operator/ComparisonOperator.py

@@ -116,16 +116,15 @@ class MatchComparisonOperator(MonovaluedComparisonOperator):
     }
     select_dict = {}
     if not only_group_columns:
-      select_dict['%s__score__' % column.replace('`', '').rsplit('.', 1)[-1]] = match_string
-    # Support sort on the relevance by using (column)__score__ key.
+      select_dict['%s__score__' % column.replace('`', '').replace('.', '_')] = match_string
     order_by_dict = {
-      '`%s__score__`' % '`.`'.join([x.strip('`') for x in column.split('.')]): match_string,
+      '%s__score__' % column.replace('`', '').replace('.', '_'): match_string
     }
     return SQLExpression(
       self,
       select_dict=select_dict,
-      where_expression=match_string,
       order_by_dict=order_by_dict,
+      where_expression=match_string,
       can_merge_select_dict=True,
     )
 

product/ZSQLCatalog/Query/EntireQuery.py

@@ -61,6 +61,7 @@ class EntireQuery(object):
                left_join_list=(),
                limit=None,
                catalog_table_name=None,
+               auto_extend_select_list=False,
                extra_column_list=(),
                from_expression=None,
                order_by_override_list=None,
@@ -76,6 +77,7 @@ class EntireQuery(object):
     self.extra_column_list = list(extra_column_list)
     self.from_expression = from_expression
     self.implicit_join = implicit_join
+    self.auto_extend_select_list = auto_extend_select_list
 
   def asSearchTextExpression(self, sql_catalog):
     return self.query.asSearchTextExpression(sql_catalog)
@@ -211,6 +213,7 @@ class EntireQuery(object):
       order_by_list=self.order_by_list,
       group_by_list=self.group_by_list,
       select_dict=self.final_select_dict,
+      auto_extend_select_list=self.auto_extend_select_list,
       limit=self.limit,
       where_expression_operator='and',
       sql_expression_list=self.sql_expression_list)

product/ZSQLCatalog/SQLCatalog.py

@@ -2350,7 +2350,8 @@ class Catalog(Folder,
     return order_by_list
 
   def buildEntireQuery(self, kw, query_table='catalog', ignore_empty_string=1,
-                       limit=None, extra_column_list=()):
+                       limit=None, auto_extend_select_list=False,
+                       extra_column_list=()):
     group_by_list = kw.pop('group_by_list', kw.pop('group_by', kw.pop('group_by_expression', ())))
     if isinstance(group_by_list, basestring):
       group_by_list = [x.strip() for x in group_by_list.split(',')]
@@ -2417,18 +2418,21 @@ class Catalog(Folder,
       implicit_join=implicit_join,
       limit=limit,
       catalog_table_name=query_table,
+      auto_extend_select_list=auto_extend_select_list,
       extra_column_list=extra_column_list,
       from_expression=from_expression)
 
   def buildSQLQuery(self, query_table='catalog', REQUEST=None,
                           ignore_empty_string=1, only_group_columns=False,
-                          limit=None, extra_column_list=(),
+                          limit=None, auto_extend_select_list=False,
+                          extra_column_list=(),
                           **kw):
     return self.buildEntireQuery(
       kw,
       query_table=query_table,
       ignore_empty_string=ignore_empty_string,
       limit=limit,
+      auto_extend_select_list=auto_extend_select_list,
       extra_column_list=extra_column_list,
     ).asSQLExpression(
       self,

product/ZSQLCatalog/SQLExpression.py

@@ -94,6 +94,7 @@ class SQLExpression(object):
                where_expression_operator=None,
                sql_expression_list=(),
                select_dict=None,
+               auto_extend_select_list=False,
                limit=None,
                from_expression=None,
                can_merge_select_dict=False):
@@ -120,6 +121,7 @@ class SQLExpression(object):
       sql_expression_list = [x for x in sql_expression_list if x is not None]
     self.sql_expression_list = list(sql_expression_list)
     self.select_dict = defaultDict(select_dict)
+    self.auto_extend_select_list = auto_extend_select_list
     if limit is None:
       self.limit = ()
     elif isinstance(limit, (list, tuple)):
@@ -133,6 +135,17 @@ class SQLExpression(object):
       warnings.warn("Providing a 'from_expression' is deprecated.",
                     DeprecationWarning)
     self.from_expression = from_expression
+    self._select_dict = self._getSelectDict()[0]
+    if self.auto_extend_select_list:
+      select_column_set = {y for x, y in self._select_dict.iteritems()}
+      extend_column_set = set(self.group_by_list).union(
+        {x[0] for x in self.order_by_list})
+      for i in extend_column_set.difference(select_column_set):
+        # '__score__' suffix alias is already added in select_dict by
+        # MatchComparisonOperator.
+        if '__score__' not in i:
+          self._select_dict['%s__ext__' % i.replace('`', '').replace('.', '_')] = i
+    self._reversed_select_dict = {y: x for x, y in self._select_dict.iteritems()}
 
   def getTableAliasDict(self):
     """
@@ -236,9 +249,8 @@ class SQLExpression(object):
     append = result.append
     order_by_dict = self._getOrderByDict()
     for (column, direction, cast) in self.getOrderByList():
-      if column.endswith('__score__') and column not in order_by_dict:
-        continue
       expression = conflictSafeGet(order_by_dict, column, str(column))
+      expression = self._reversed_select_dict.get(expression, expression)
       if cast not in (None, ''):
         expression = 'CAST(%s AS %s)' % (expression, cast)
       if direction is not None:
@@ -304,7 +316,7 @@ class SQLExpression(object):
       If there are nested SQLExpression, it merges (union of sets) them with
       local value.
     """
-    result = set(self.group_by_list)
+    result = {self._reversed_select_dict.get(x, x) for x in self.group_by_list}
     for sql_expression in self.sql_expression_list:
       result.update(sql_expression.getGroupByset())
     return result
@@ -363,7 +375,7 @@ class SQLExpression(object):
       checks that they don't alias different columns with the same name. If
       they do, it raises a ValueError.
     """
-    return self._getSelectDict()[0]
+    return self._select_dict
 
   def getSelectExpression(self):
     """

product/ZSQLCatalog/interfaces/entire_query.py

@@ -45,6 +45,7 @@ class IEntireQuery(Interface):
 
   def __init__(query, order_by_list=None, group_by_list=None,
     select_dict=None, limit=None, catalog_table_name=None,
+    auto_extend_select_list=False,
     extra_column_list=None, from_expression=None,
     order_by_override_list=None):
     """
@@ -67,6 +68,11 @@ class IEntireQuery(Interface):
         See SQLExpression.
       catalog_table_name (string)
         Name of the table to use as a catalog.
+      auto_extend_select_list (boolean)
+        If True, select_list is automatically extended to have columns
+        used in group_by_list and order_by_list. It is useful when use
+        select_expression in inner query and use group_by_expression or
+        order_by_expression in outer query.
 
       Deprecated parameters.
       extra_column_list (list of string)

product/ZSQLCatalog/interfaces/query_catalog.py

@@ -64,7 +64,8 @@ class ISearchKeyCatalog(Interface):
     """
 
   def buildEntireQuery(kw, query_table='catalog', ignore_empty_string=1,
-                       limit=None, extra_column_list=None):
+                       limit=None, auto_extend_select_list=False,
+                       extra_column_list=None):
     """
       Construct and return an instance of EntireQuery class from given
       parameters by calling buildQuery.
@@ -95,6 +96,11 @@ class ISearchKeyCatalog(Interface):
          - type cast (see SQL documentation of 'CAST')
         Sort will happen on given parameter name (its column if it's a column
         name, corresponding virtual column otherwise - as for related keys).
+      auto_extend_select_list (boolean)
+        If True, select_list is automatically extended to have columns
+        used in group_by_list and order_by_list. It is useful when use
+        select_expression in inner query and use group_by_expression or
+        order_by_expression in outer query.
       Extra parameters are passed through to buildQuery.
 
       Backward compatibility parameters:
@@ -140,7 +146,8 @@ class ISearchKeyCatalog(Interface):
 
   def buildSQLQuery(query_table='catalog', REQUEST=None,
                     ignore_empty_string=1, only_group_columns=False,
-                    limit=None, extra_column_list=None,
+                    limit=None, auto_extend_select_list=False,
+                    extra_column_list=(),
                     **kw):
     """
       Return an SQLExpression-generated dictionary (see

product/ZSQLCatalog/interfaces/sql_expression.py

@@ -61,6 +61,7 @@ class ISQLExpression(Interface):
                where_expression_operator=None,
                sql_expression_list=None,
                select_dict=None,
+               auto_extend_select_list=False,
                limit=None,
                from_expression=None):
     """
@@ -100,6 +101,11 @@ class ISQLExpression(Interface):
         Key is column alias.
         Value is column name, or Null. If it is Null, the alias will also be
         used as column name.
+      auto_extend_select_list (boolean)
+        If True, select_list is automatically extended to have columns
+        used in group_by_list and order_by_list. It is useful when use
+        select_expression in inner query and use group_by_expression or
+        order_by_expression in outer query.
       limit (1-tuple, 2-tuple, other)
         First item is the number of lines expected, second one if given is the
         offset of limited result list within the unlimited result list.

product/ZSQLCatalog/tests/testSQLCatalog.py

@@ -728,27 +728,27 @@ class TestSQLCatalog(ERP5TypeTestCase):
     order_by_expression = sql_expression.getOrderByExpression()
     self.assertNotEqual(order_by_expression, '')
     # ... and not sort by relevance
-    self.assertFalse('MATCH' in order_by_expression, order_by_expression)
+    self.assertEqual('`foo`.`fulltext`', order_by_expression)
     # order_by_list on fulltext column + '__score__, resulting "ORDER BY" must be non-empty.
     sql_expression = self.asSQLExpression({'fulltext': 'foo',
       'order_by_list': [('fulltext__score__', ), ]})
     order_by_expression = sql_expression.getOrderByExpression()
     self.assertNotEqual(order_by_expression, '')
     # ... and must sort by relevance
-    self.assertTrue('MATCH' in order_by_expression, order_by_expression)
+    self.assertEqual('foo_fulltext__score__', order_by_expression)
     # ordering on fulltext column with sort order specified must preserve
     # sorting by relevance.
     for direction in ('ASC', 'DESC'):
       sql_expression = self.asSQLExpression({'fulltext': 'foo',
         'order_by_list': [('fulltext__score__', direction), ]})
       order_by_expression = sql_expression.getOrderByExpression()
-      self.assertTrue('MATCH' in order_by_expression, (order_by_expression, direction))
+      self.assertEqual('foo_fulltext__score__ %s' % direction, order_by_expression)
     # Providing a None cast should work too
     for direction in ('ASC', 'DESC'):
       sql_expression = self.asSQLExpression({'fulltext': 'foo',
         'order_by_list': [('fulltext__score__', direction, None), ]})
       order_by_expression = sql_expression.getOrderByExpression()
-      self.assertTrue('MATCH' in order_by_expression, (order_by_expression, direction))
+      self.assertEqual('foo_fulltext__score__ %s' % direction, order_by_expression)
 
   def test_logicalOperators(self):
     self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='AN ORB'),
@@ -759,6 +759,42 @@ class TestSQLCatalog(ERP5TypeTestCase):
         operator='and'),
       {'default': 'AN OR ORB'})
 
+  def test_auto_extend_select_list(self):
+    # by default select_list is not automatically extended by
+    # order_by_list or group_by_list.
+    sql_expression = self.asSQLExpression({
+      'order_by_list': [('default',),]})
+    select_dict = sql_expression.getSelectDict()
+    self.assertEqual({}, select_dict)
+    sql_expression = self.asSQLExpression({
+      'group_by_list': ['default',]})
+    select_dict = sql_expression.getSelectDict()
+    self.assertEqual({}, select_dict)
+    # select_list is extended if auto_extend_select_list is enabled.
+    sql_expression = self.asSQLExpression({
+      'order_by_list': [('default',),]},
+      auto_extend_select_list=True)
+    select_dict = sql_expression.getSelectDict()
+    self.assertEqual({'foo_default__ext__': '`foo`.`default`'}, select_dict)
+    sql_expression = self.asSQLExpression({
+      'group_by_list': ['default',]},
+      auto_extend_select_list=True)
+    select_dict = sql_expression.getSelectDict()
+    self.assertEqual({'foo_default__ext__': '`foo`.`default`'}, select_dict)
+    # fulltext score is automatically added in select_dict even if
+    # auto_extend_select_list is not enabled.
+    sql_expression = self.asSQLExpression({
+      'fulltext': 'foo',
+      'order_by_list': [('fulltext__score__',),]})
+    select_dict = sql_expression.getSelectDict()
+    self.assertEqual(['foo_fulltext__score__'], select_dict.keys())
+    sql_expression = self.asSQLExpression({
+      'fulltext': 'foo',
+      'order_by_list': [('fulltext__score__',),]},
+      auto_extend_select_list=True)
+    select_dict = sql_expression.getSelectDict()
+    self.assertEqual(['foo_fulltext__score__'], select_dict.keys())
+
   def _searchTextInDictQuery(self, column):
     self.catalog(ReferenceQuery(ReferenceQuery(
         ReferenceQuery(operator='>=', date=DateTime('2001/08/11')),