backport of bug #54461 from 5.1-security to 5.0-security

 > revision-id: gshchepa@mysql.com-20100801181236-uyuq6ewaq43rw780
 > parent: alexey.kopytov@sun.com-20100723115254-jjwmhq97b9wl932l
 > committer: Gleb Shchepa <gshchepa@mysql.com>
 > branch nick: mysql-5.1-security
 > timestamp: Sun 2010-08-01 22:12:36 +0400
 > Bug #54461: crash with longblob and union or update with subquery
 >
 > Queries may crash, if
 >   1) the GREATEST or the LEAST function has a mixed list of
 >      numeric and LONGBLOB arguments and
 >   2) the result of such a function goes through an intermediate
 >      temporary table.
 >
 > An Item that references a LONGBLOB field has max_length of
 > UINT_MAX32 == (2^32 - 1).
 >
 > The current implementation of GREATEST/LEAST returns REAL
 > result for a mixed list of numeric and string arguments (that
 > contradicts with the current documentation, this contradiction
 > was discussed and it was decided to update the documentation).
 >
 > The max_length of such a function call was calculated as a
 > maximum of argument max_length values (i.e. UINT_MAX32).
 >
 > That max_length value of UINT_MAX32 was used as a length for
 > the intermediate temporary table Field_double to hold
 > GREATEST/LEAST function result.
 >
 > The Field_double::val_str() method call on that field
 > allocates a String value.
 >
 > Since an allocation of String reserves an additional byte
 > for a zero-termination, the size of String buffer was
 > set to (UINT_MAX32 + 1), that caused an integer overflow:
 > actually, an empty buffer of size 0 was allocated.
 >
 > An initialization of the "first" byte of that zero-size
 > buffer with '\0' caused a crash.
 >
 > The Item_func_min_max::fix_length_and_dec() has been
 > modified to calculate max_length for the REAL result like
 > we do it for arithmetical operators.


mysql-test/r/func_misc.result:
  Test case for bug #54461.
mysql-test/t/func_misc.test:
  Test case for bug #54461.
sql/item_func.cc:
  Bug #54461: crash with longblob and union or update with subquery
  
  The Item_func_min_max::fix_length_and_dec() has been
  modified to calculate max_length for the REAL result like
  we do it for arithmetical operators.

mysql-test/r/func_misc.result

@@ -327,4 +327,19 @@ DROP TABLE t1;
 select NAME_CONST('_id',1234) as id;
 id
 1234
+#
+# Bug #54461: crash with longblob and union or update with subquery
+#
+CREATE TABLE t1 (a INT, b LONGBLOB);
+INSERT INTO t1 VALUES (1, '2'), (2, '3'), (3, '2');
+SELECT DISTINCT LEAST(a, (SELECT b FROM t1 LIMIT 1)) FROM t1 UNION SELECT 1;
+LEAST(a, (SELECT b FROM t1 LIMIT 1))
+1
+2
+SELECT DISTINCT GREATEST(a, (SELECT b FROM t1 LIMIT 1)) FROM t1 UNION SELECT 1;
+GREATEST(a, (SELECT b FROM t1 LIMIT 1))
+2
+3
+1
+DROP TABLE t1;
 End of 5.0 tests

mysql-test/t/func_misc.test

@@ -450,5 +450,16 @@ DROP TABLE t1;
 #
 select NAME_CONST('_id',1234) as id;
 
---echo End of 5.0 tests
+--echo #
+--echo # Bug #54461: crash with longblob and union or update with subquery
+--echo #
+
+CREATE TABLE t1 (a INT, b LONGBLOB);
+INSERT INTO t1 VALUES (1, '2'), (2, '3'), (3, '2');
+
+SELECT DISTINCT LEAST(a, (SELECT b FROM t1 LIMIT 1)) FROM t1 UNION SELECT 1;
+SELECT DISTINCT GREATEST(a, (SELECT b FROM t1 LIMIT 1)) FROM t1 UNION SELECT 1;
 
+DROP TABLE t1;
+
+--echo End of 5.0 tests

sql/item_func.cc

@@ -2250,6 +2250,8 @@ void Item_func_min_max::fix_length_and_dec()
   else if ((cmp_type == DECIMAL_RESULT) || (cmp_type == INT_RESULT))
     max_length= my_decimal_precision_to_length(max_int_part+decimals, decimals,
                                             unsigned_flag);
+  else if (cmp_type == REAL_RESULT)
+    max_length= float_length(decimals);
   cached_field_type= agg_field_type(args, arg_count);
 }