Fixed bug#15351: Wrong collation used for comparison of md5() and sha()

argument can lead to a wrong result.

md5() and sha() functions treat their arguments as case sensitive strings.
But when they are compared their arguments were compared as a case
insensitive strings which leads to two functions with different arguments
and thus different results to being identical. This can lead to a wrong
decision made in the range optimizer and thus lead to a wrong result set.

Item_func_md5::fix_length_and_dec() and Item_func_sha::fix_length_and_dec()
functions now set binary collation on their arguments.

mysql-test/r/func_str.result

@@ -1006,4 +1006,19 @@ NULL
 select ifnull(load_file("lkjlkj"),"it's null");
 ifnull(load_file("lkjlkj"),"it's null")
 it's null
+create table t1 (f1 varchar(4), f2 varchar(64), unique key k1 (f1,f2));
+insert into t1 values ( 'test',md5('test')), ('test', sha('test'));
+select * from t1 where f1='test' and (f2= md5("test") or f2= md5("TEST"));
+f1	f2
+test	098f6bcd4621d373cade4e832627b4f6
+select * from t1 where f1='test' and (f2= md5("TEST") or f2= md5("test"));
+f1	f2
+test	098f6bcd4621d373cade4e832627b4f6
+select * from t1 where f1='test' and (f2= sha("test") or f2= sha("TEST"));
+f1	f2
+test	a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
+select * from t1 where f1='test' and (f2= sha("TEST") or f2= sha("test"));
+f1	f2
+test	a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
+drop table t1;
 End of 4.1 tests

mysql-test/t/func_str.test

@@ -669,4 +669,16 @@ drop table t1;
 select load_file("lkjlkj");
 select ifnull(load_file("lkjlkj"),"it's null");
 
+#
+# Bug#15351: Wrong collation used for comparison of md5() and sha()
+# parameter can lead to a wrong result.
+#
+create table t1 (f1 varchar(4), f2 varchar(64), unique key k1 (f1,f2));
+insert into t1 values ( 'test',md5('test')), ('test', sha('test'));
+select * from t1 where f1='test' and (f2= md5("test") or f2= md5("TEST"));
+select * from t1 where f1='test' and (f2= md5("TEST") or f2= md5("test"));
+select * from t1 where f1='test' and (f2= sha("test") or f2= sha("TEST"));
+select * from t1 where f1='test' and (f2= sha("TEST") or f2= sha("test"));
+drop table t1;
+
 --echo End of 4.1 tests

sql/item_strfunc.cc

@@ -118,7 +118,15 @@ String *Item_func_md5::val_str(String *str)
 
 void Item_func_md5::fix_length_and_dec()
 {
-   max_length=32;
+  max_length=32;
+  /*
+    The MD5() function treats its parameter as being a case sensitive. Thus
+    we set binary collation on it so different instances of MD5() will be
+    compared properly.
+  */
+  args[0]->collation.set(
+      get_charset_by_csname(args[0]->collation.collation->csname,
+                            MY_CS_BINSORT,MYF(0)), DERIVATION_COERCIBLE);
 }
 
 
@@ -159,7 +167,15 @@ String *Item_func_sha::val_str(String *str)
 
 void Item_func_sha::fix_length_and_dec()
 {
-   max_length=SHA1_HASH_SIZE*2; // size of hex representation of hash
+  max_length=SHA1_HASH_SIZE*2; // size of hex representation of hash
+  /*
+    The SHA() function treats its parameter as being a case sensitive. Thus
+    we set binary collation on it so different instances of MD5() will be
+    compared properly.
+  */
+  args[0]->collation.set(
+      get_charset_by_csname(args[0]->collation.collation->csname,
+                            MY_CS_BINSORT,MYF(0)), DERIVATION_COERCIBLE);
 }