Bug #20536: md5() with GROUP BY and UCS2 return different results on myisam/innodb

Make the encryption functions MD5(), SHA1() and ENCRYPT() return binary results.
    
Make MAKE_SET() and EXPORT_SET() use the correct character set for their default separator strings.

mysql-test/r/ctype_ucs.result

@@ -722,3 +722,46 @@ id	MIN(s)
 1	ZZZ
 2	ZZZ
 DROP TABLE t1;
+drop table if exists bug20536;
+set names latin1;
+create table bug20536 (id bigint not null auto_increment primary key, name
+varchar(255) character set ucs2 not null);
+insert into `bug20536` (`id`,`name`) values (1, _latin1 x'74657374311a'), (2, "'test\\_2'");
+select md5(name) from bug20536;
+md5(name)
+3417d830fe24ffb2f81a28e54df2d1b3
+48d95db0d8305c2fe11548a3635c9385
+select sha1(name) from bug20536;
+sha1(name)
+72228a6d56efb7a89a09543068d5d8fa4c330881
+677d4d505355eb5b0549b865fcae4b7f0c28aef5
+select make_set(3, name, upper(name)) from bug20536;
+make_set(3, name, upper(name))
+test1,TEST1
+'test\_2','TEST\_2'
+select export_set(5, name, upper(name)) from bug20536;
+export_set(5, name, upper(name))
+test1,TEST1,test1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1,TEST1
+'test\_2','TEST\_2','test\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2','TEST\_2'
+select export_set(5, name, upper(name), ",", 5) from bug20536;
+export_set(5, name, upper(name), ",", 5)
+test1,TEST1,test1,TEST1,TEST1
+'test\_2','TEST\_2','test\_2','TEST\_2','TEST\_2'
+select password(name) from bug20536;
+password(name)
+????????????????????
+????????????????????
+select old_password(name) from bug20536;
+old_password(name)
+????????
+????????
+select encrypt(name, 'SALT') from bug20536;
+encrypt(name, 'SALT')
+SA5pDi1UPZdys
+SA5pDi1UPZdys
+select quote(name) from bug20536;
+quote(name)
+??????????
+????????????????
+drop table bug20536;
+End of 4.1 tests

mysql-test/t/ctype_ucs.test

@@ -463,4 +463,43 @@ INSERT INTO t1 VALUES (1, 'ZZZZZ'), (1, 'ZZZ'), (2, 'ZZZ'), (2, 'ZZZZZ');
 SELECT id, MIN(s) FROM t1 GROUP BY id;
 
 DROP TABLE t1;
-# End of 4.1 tests
+
+#
+# Bug #20536: md5() with GROUP BY and UCS2 return different results on myisam/innodb
+#
+
+--disable_warnings
+drop table if exists bug20536;
+--enable_warnings
+
+set names latin1;
+create table bug20536 (id bigint not null auto_increment primary key, name
+varchar(255) character set ucs2 not null);
+insert into `bug20536` (`id`,`name`) values (1, _latin1 x'74657374311a'), (2, "'test\\_2'");
+select md5(name) from bug20536;
+select sha1(name) from bug20536;
+select make_set(3, name, upper(name)) from bug20536;
+select export_set(5, name, upper(name)) from bug20536;
+select export_set(5, name, upper(name), ",", 5) from bug20536;
+
+# Some broken functions:  add these tests just to document current behavior.
+
+# PASSWORD and OLD_PASSWORD don't work with UCS2 strings, but to fix it would
+# not be backwards compatible in all cases, so it's best to leave it alone
+select password(name) from bug20536;
+select old_password(name) from bug20536;
+
+# ENCRYPT relies on OS function crypt() which takes a NUL-terminated string; it
+# doesn't return good results for strings with embedded 0 bytes.  It won't be
+# fixed unless we choose to re-implement the crypt() function ourselves to take
+# an extra size_t string_length argument.
+select encrypt(name, 'SALT') from bug20536;
+
+# QUOTE doesn't work with UCS2 data.  It would require a total rewrite
+# of Item_func_quote::val_str(), which isn't worthwhile until UCS2 is
+# supported fully as a client character set.
+select quote(name) from bug20536;
+
+drop table bug20536;
+
+--echo End of 4.1 tests

sql/item_strfunc.cc

@@ -88,6 +88,7 @@ String *Item_func_md5::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   String * sptr= args[0]->val_str(str);
+  str->set_charset(&my_charset_bin);
   if (sptr)
   {
     my_MD5_CTX context;
@@ -134,6 +135,7 @@ String *Item_func_sha::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   String * sptr= args[0]->val_str(str);
+  str->set_charset(&my_charset_bin);
   if (sptr)  /* If we got value different from NULL */
   {
     SHA1_CONTEXT context;  /* Context used to generate SHA1 hash */
@@ -1529,7 +1531,7 @@ String *Item_func_encrypt::val_str(String *str)
     null_value= 1;
     return 0;
   }
-  str->set(tmp,(uint) strlen(tmp),res->charset());
+  str->set(tmp, (uint) strlen(tmp), &my_charset_bin);
   str->copy();
   pthread_mutex_unlock(&LOCK_crypt);
   return str;
@@ -1926,7 +1928,7 @@ String *Item_func_make_set::val_str(String *str)
 	      return &my_empty_string;
 	    result= &tmp_str;
 	  }
-	  if (tmp_str.append(',') || tmp_str.append(*res))
+	  if (tmp_str.append(",", 1, &my_charset_bin) || tmp_str.append(*res))
 	    return &my_empty_string;
 	}
       }
@@ -2592,8 +2594,12 @@ String* Item_func_export_set::val_str(String* str)
     }
     break;
   case 3:
-    sep_buf.set(",", 1, default_charset());
-    sep = &sep_buf;
+    {
+      /* errors is not checked - assume "," can always be converted */
+      uint errors;
+      sep_buf.copy(",", 1, &my_charset_bin, collation.collation, &errors);
+      sep = &sep_buf;
+    }
     break;
   default:
     DBUG_ASSERT(0); // cannot happen

sql/item_strfunc.h

@@ -41,7 +41,10 @@ class Item_func_md5 :public Item_str_func
 {
   String tmp_value;
 public:
-  Item_func_md5(Item *a) :Item_str_func(a) {}
+  Item_func_md5(Item *a) :Item_str_func(a)
+  {
+    collation.set(&my_charset_bin);
+  }
   String *val_str(String *);
   void fix_length_and_dec();
   const char *func_name() const { return "md5"; }
@@ -51,7 +54,10 @@ class Item_func_md5 :public Item_str_func
 class Item_func_sha :public Item_str_func
 {
 public:
-  Item_func_sha(Item *a) :Item_str_func(a) {}  
+  Item_func_sha(Item *a) :Item_str_func(a)
+  {
+    collation.set(&my_charset_bin);
+  }
   String *val_str(String *);    
   void fix_length_and_dec();      
   const char *func_name() const { return "sha"; }	
@@ -306,9 +312,21 @@ class Item_func_des_decrypt :public Item_str_func
 class Item_func_encrypt :public Item_str_func
 {
   String tmp_value;
+
+  /* Encapsulate common constructor actions */
+  void constructor_helper()
+  {
+    collation.set(&my_charset_bin);
+  }
 public:
-  Item_func_encrypt(Item *a) :Item_str_func(a) {}
-  Item_func_encrypt(Item *a, Item *b): Item_str_func(a,b) {}
+  Item_func_encrypt(Item *a) :Item_str_func(a)
+  {
+    constructor_helper();
+  }
+  Item_func_encrypt(Item *a, Item *b): Item_str_func(a,b)
+  {
+    constructor_helper();
+  }
   String *val_str(String *);
   void fix_length_and_dec() { maybe_null=1; max_length = 13; }
   const char *func_name() const { return "ecrypt"; }