Fix for bug#4488: sign-aware equality check

mysql-test/r/range.result

@@ -500,6 +500,9 @@ count(*)
 select count(*) from t1 where x > -16;
 count(*)
 2
+select count(*) from t1 where x = 18446744073709551601;
+count(*)
+1
 create table t2 (x bigint not null);
 insert into t2(x) values (0xfffffffffffffff0);
 insert into t2(x) values (0xfffffffffffffff1);
@@ -525,6 +528,9 @@ count(*)
 select count(*) from t2 where x > -16;
 count(*)
 1
+select count(*) from t2 where x = 18446744073709551601;
+count(*)
+0
 drop table t1;
 create table t1 (x bigint unsigned not null primary key) engine=innodb;
 insert into t1(x) values (0xfffffffffffffff0);
@@ -551,4 +557,7 @@ count(*)
 select count(*) from t1 where x > -16;
 count(*)
 1
+select count(*) from t1 where x = 18446744073709551601;
+count(*)
+1
 drop table t1;

mysql-test/t/range.test

@@ -396,6 +396,8 @@ select count(*) from t1 where x<0;
 select count(*) from t1 where x < -16;
 select count(*) from t1 where x = -16;
 select count(*) from t1 where x > -16;
+select count(*) from t1 where x = 18446744073709551601;
+
 
 create table t2 (x bigint not null);
 insert into t2(x) values (0xfffffffffffffff0);
@@ -407,6 +409,7 @@ select count(*) from t2 where x<0;
 select count(*) from t2 where x < -16;
 select count(*) from t2 where x = -16;
 select count(*) from t2 where x > -16;
+select count(*) from t2 where x = 18446744073709551601;
 
 drop table t1;
 create table t1 (x bigint unsigned not null primary key) engine=innodb;
@@ -419,6 +422,7 @@ select count(*) from t1 where x<0;
 select count(*) from t1 where x < -16;
 select count(*) from t1 where x = -16;
 select count(*) from t1 where x > -16;
+select count(*) from t1 where x = 18446744073709551601;
 
 drop table t1;
 

sql/item_cmpfunc.cc

@@ -325,6 +325,11 @@ int Arg_comparator::set_compare_func(Item_bool_func2 *item, Item_result type)
       else if ((*b)->unsigned_flag)
         func= &Arg_comparator::compare_int_signed_unsigned;
     }
+    else if (func== &Arg_comparator::compare_e_int)
+    {
+      if ((*a)->unsigned_flag ^ (*b)->unsigned_flag)
+        func= &Arg_comparator::compare_e_int_diff_signedness;
+    }
   }
   return 0;
 }
@@ -530,6 +535,17 @@ int Arg_comparator::compare_e_int()
   return test(val1 == val2);
 }
 
+/*
+  Compare unsigned *a with signed *b or signed *a with unsigned *b.
+*/
+int Arg_comparator::compare_e_int_diff_signedness()
+{
+  longlong val1= (*a)->val_int();
+  longlong val2= (*b)->val_int();
+  if ((*a)->null_value || (*b)->null_value)
+    return test((*a)->null_value && (*b)->null_value);
+  return (val1 >= 0) && test(val1 == val2);
+}
 
 int Arg_comparator::compare_row()
 {

sql/item_cmpfunc.h

@@ -74,6 +74,7 @@ class Arg_comparator: public Sql_alloc
   int compare_e_binary_string(); // compare args[0] & args[1]
   int compare_e_real();          // compare args[0] & args[1]
   int compare_e_int();           // compare args[0] & args[1]
+  int compare_e_int_diff_signedness();
   int compare_e_row();           // compare args[0] & args[1]
 
   static arg_cmp_func comparator_matrix [4][2];