Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT

When temporary tables is used for result sorting
result field for gconcat function is created using
group_concat_max_len size. It leads to result truncation
when character_set_results is multi-byte character set due
to insufficient tmp table field size.
The fix is to increase temporary table field size for
gconcat. Method make_string_field() is overloaded
for Item_func_group_concat class and uses
max_characters * collation.collation->mbmaxlen size for
result field. max_characters is maximum number of characters
what can fit into max_length size.


mysql-test/r/ctype_utf16.result:
  test result
mysql-test/r/ctype_utf32.result:
  test result
mysql-test/r/ctype_utf8.result:
  test result
mysql-test/t/ctype_utf16.test:
  test case
mysql-test/t/ctype_utf32.test:
  test case
mysql-test/t/ctype_utf8.test:
  test case
sql/item.h:
  make Item::make_string_field() virtual
sql/item_sum.cc:
  added Item_func_group_concat::make_string_field(TABLE *table) method
  which uses max_characters * collation.collation->mbmaxlen size for
  result item. max_characters is maximum number of characters what can
  fit into max_length size.
sql/item_sum.h:
  added Item_func_group_concat::make_string_field(TABLE *table) method

mysql-test/r/ctype_utf16.result

@@ -1126,5 +1126,19 @@ NULL
 Warnings:
 Warning	1301	Result of repeat() was larger than max_allowed_packet (1048576) - truncated
 #
+# Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT
+#
+SET NAMES utf8, @@character_set_connection=utf16;
+SELECT id, CHAR_LENGTH(GROUP_CONCAT(body)) AS l
+FROM (SELECT 'a' AS id, REPEAT('foo bar', 100) AS body
+UNION ALL
+SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1
+GROUP BY id
+ORDER BY l DESC;
+id	l
+a	512
+Warnings:
+Warning	1260	Row 1 was cut by GROUP_CONCAT()
+#
 # End of 5.5 tests
 #

mysql-test/r/ctype_utf32.result

@@ -1167,5 +1167,19 @@ CASE s1 WHEN 'a' THEN 'b' ELSE 'c' END
 b
 DROP TABLE t1;
 #
+# Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT
+#
+SET NAMES utf8, @@character_set_connection=utf32;
+SELECT id, CHAR_LENGTH(GROUP_CONCAT(body)) AS l
+FROM (SELECT 'a' AS id, REPEAT('foo bar', 100) AS body
+UNION ALL
+SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1
+GROUP BY id
+ORDER BY l DESC;
+id	l
+a	256
+Warnings:
+Warning	1260	Row 1 was cut by GROUP_CONCAT()
+#
 # End of 5.5 tests
 #

mysql-test/r/ctype_utf8.result

@@ -2788,7 +2788,7 @@ create table t1 as select group_concat(1,2,3) as c1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varchar(342) CHARACTER SET utf8 DEFAULT NULL
+  `c1` text CHARACTER SET utf8
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
 create table t1 as select 1 as c1 union select 'a';
@@ -5010,5 +5010,19 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	select 'abcdÁÂÃÄÅ' AS `abcdÁÂÃÄÅ`,_latin1'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÁÂÃÄÅ`,_utf8'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÁÂÃÄÅ`
 #
+# Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT
+#
+SET NAMES utf8;
+SELECT id, CHAR_LENGTH(GROUP_CONCAT(body)) AS l
+FROM (SELECT 'a' AS id, REPEAT('foo bar', 100) AS body
+UNION ALL
+SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1
+GROUP BY id
+ORDER BY l DESC;
+id	l
+a	1024
+Warnings:
+Warning	1260	Row 2 was cut by GROUP_CONCAT()
+#
 # End of 5.5 tests
 #

mysql-test/t/ctype_utf16.test

@@ -762,6 +762,20 @@ DROP TABLE t1;
 
 SELECT space(date_add(101, INTERVAL CHAR('1' USING utf16) hour_second)); 
 
+
+--echo #
+--echo # Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT
+--echo #
+
+SET NAMES utf8, @@character_set_connection=utf16;
+SELECT id, CHAR_LENGTH(GROUP_CONCAT(body)) AS l
+FROM (SELECT 'a' AS id, REPEAT('foo bar', 100) AS body
+UNION ALL
+SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1
+GROUP BY id
+ORDER BY l DESC;
+
+
 #
 ## TODO: add tests for all engines
 #

mysql-test/t/ctype_utf32.test

@@ -840,6 +840,19 @@ INSERT INTO t1 VALUES ('a');
 SELECT CASE s1 WHEN 'a' THEN 'b' ELSE 'c' END FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT
+--echo #
+
+SET NAMES utf8, @@character_set_connection=utf32;
+SELECT id, CHAR_LENGTH(GROUP_CONCAT(body)) AS l
+FROM (SELECT 'a' AS id, REPEAT('foo bar', 100) AS body
+UNION ALL
+SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1
+GROUP BY id
+ORDER BY l DESC;
+
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

mysql-test/t/ctype_utf8.test

@@ -1561,6 +1561,18 @@ EXPLAIN EXTENDED SELECT 'abcdÁÂÃÄÅ', _latin1'abcdÁÂÃÄÅ', _utf8'abcdÁ
 SET NAMES utf8;
 EXPLAIN EXTENDED SELECT 'abcdÁÂÃÄÅ', _latin1'abcdÁÂÃÄÅ', _utf8'abcdÁÂÃÄÅ';
 
+--echo #
+--echo # Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT
+--echo #
+
+SET NAMES utf8;
+SELECT id, CHAR_LENGTH(GROUP_CONCAT(body)) AS l
+FROM (SELECT 'a' AS id, REPEAT('foo bar', 100) AS body
+UNION ALL
+SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1
+GROUP BY id
+ORDER BY l DESC;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

sql/item.h

@@ -592,7 +592,7 @@ class Item {
   void init_make_field(Send_field *tmp_field,enum enum_field_types type);
   virtual void cleanup();
   virtual void make_field(Send_field *field);
-  Field *make_string_field(TABLE *table);
+  virtual Field *make_string_field(TABLE *table);
   virtual bool fix_fields(THD *, Item **);
   /*
     should be used in case where we are sure that we do not need

sql/item_sum.cc

@@ -3128,6 +3128,31 @@ void Item_func_group_concat::cleanup()
 }
 
 
+Field *Item_func_group_concat::make_string_field(TABLE *table)
+{
+  Field *field;
+  DBUG_ASSERT(collation.collation);
+  /*
+    max_characters is maximum number of characters
+    what can fit into max_length size. It's necessary
+    to use field size what allows to store group_concat
+    result without truncation. For this purpose we use
+    max_characters * CS->mbmaxlen.
+  */
+  const uint32 max_characters= max_length / collation.collation->mbminlen;
+  if (max_characters > CONVERT_IF_BIGGER_TO_BLOB)
+    field= new Field_blob(max_characters * collation.collation->mbmaxlen,
+                          maybe_null, name, collation.collation, TRUE);
+  else
+    field= new Field_varstring(max_characters * collation.collation->mbmaxlen,
+                               maybe_null, name, table->s, collation.collation);
+
+  if (field)
+    field->init(table);
+  return field;
+}
+
+
 Item *Item_func_group_concat::copy_or_same(THD* thd)
 {
   return new (thd->mem_root) Item_func_group_concat(thd, this);

sql/item_sum.h

@@ -1411,6 +1411,7 @@ class Item_func_group_concat : public Item_sum
   enum Sumfunctype sum_func () const {return GROUP_CONCAT_FUNC;}
   const char *func_name() const { return "group_concat"; }
   virtual Item_result result_type () const { return STRING_RESULT; }
+  virtual Field *make_string_field(TABLE *table);
   enum_field_types field_type() const
   {
     if (max_length/collation.collation->mbmaxlen > CONVERT_IF_BIGGER_TO_BLOB )