Merge spetrunia@bk-internal.mysql.com:/home/bk/mysql-5.0

into mysql.com:/home/psergey/mysql-5.0-bug8397

mysql-test/r/type_decimal.result

-DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t1, t2;
 SET SQL_WARNINGS=1;
 CREATE TABLE t1 (
 id int(11) NOT NULL auto_increment,
@@ -677,3 +677,19 @@ a
 9999.999
 0000.000
 drop table t1;
+CREATE TABLE t1
+(EMPNUM   CHAR(3) NOT NULL,
+HOURS    DECIMAL(5));
+CREATE TABLE t2
+(EMPNUM   CHAR(3) NOT NULL,
+HOURS    BIGINT);
+INSERT INTO t1 VALUES ('E1',40);
+INSERT INTO t1 VALUES ('E8',NULL);
+INSERT INTO t2 VALUES ('E1',40);
+SELECT EMPNUM FROM t1 WHERE HOURS IN (SELECT HOURS FROM t2);
+EMPNUM
+E1
+SELECT EMPNUM FROM t1 WHERE HOURS IN (SELECT HOURS FROM t1);
+EMPNUM
+E1
+DROP TABLE t1,t2;

mysql-test/t/type_decimal.test

 # bug in decimal() with negative numbers by kaido@tradenet.ee
 
 --disable_warnings
-DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t1, t2;
 --enable_warnings
 SET SQL_WARNINGS=1;
 
@@ -268,3 +268,20 @@ insert into t1 values ('1'),('+1'),('-1'),('0000000001'),('+0000000001'),('-0000
 --enable_warnings
 select * from t1;
 drop table t1;
+
+# Test for BUG#8397: decimal type in subselects (Item_cache_decimal)
+CREATE TABLE t1
+(EMPNUM   CHAR(3) NOT NULL,
+HOURS    DECIMAL(5));
+CREATE TABLE t2
+(EMPNUM   CHAR(3) NOT NULL,
+HOURS    BIGINT);
+
+INSERT INTO t1 VALUES ('E1',40);
+INSERT INTO t1 VALUES ('E8',NULL);
+INSERT INTO t2 VALUES ('E1',40);
+
+SELECT EMPNUM FROM t1 WHERE HOURS IN (SELECT HOURS FROM t2);
+SELECT EMPNUM FROM t1 WHERE HOURS IN (SELECT HOURS FROM t1);
+
+DROP TABLE t1,t2;

sql/item.cc

@@ -4349,7 +4349,7 @@ my_decimal *Item_cache_real::val_decimal(my_decimal *decimal_val)
 void Item_cache_decimal::store(Item *item)
 {
   my_decimal *val= item->val_decimal_result(&decimal_value);
-  if (val != &decimal_value)
+  if (val != &decimal_value && !item->null_value)
     my_decimal2decimal(val, &decimal_value);
   null_value= item->null_value;
 }

sql/opt_range.cc

@@ -2562,8 +2562,8 @@ static double ror_scan_selectivity(const ROR_INTERSECT_INFO *info,
   char *key_ptr= (char*) key_val;
   SEL_ARG *sel_arg, *tuple_arg= NULL;
   bool cur_covered;
-  bool prev_covered= (bitmap_is_set(&info->covered_fields, 
-                                    key_part->fieldnr))? 1 : 0;
+  bool prev_covered= test(bitmap_is_set(&info->covered_fields,
+                                        key_part->fieldnr));
   key_range min_range;
   key_range max_range;
   min_range.key= (byte*) key_val;
@@ -2577,8 +2577,8 @@ static double ror_scan_selectivity(const ROR_INTERSECT_INFO *info,
       i++, sel_arg= sel_arg->next_key_part)
   {
     DBUG_PRINT("info",("sel_arg step"));
-    cur_covered= (bitmap_is_set(&info->covered_fields,
-                                (key_part + i)->fieldnr))? 1 : 0;
+    cur_covered= test(bitmap_is_set(&info->covered_fields,
+                                    (key_part + i)->fieldnr));
     if (cur_covered != prev_covered)
     {
       /* create (part1val, ..., part{n-1}val) tuple. */