Bug#30889: filesort and order by with float/numeric crashes server

There are two problems with ROUND(X, D) on an exact numeric 
(DECIMAL, NUMERIC type) field of a table:
1) The implementation of the ROUND function would change the number of decimal
places regardless of the value decided upon in fix_length_and_dec. When the
number of decimal places is not constant, this would cause an inconsistent
state where the number of digits was less than the number of decimal places,
which crashes filesort.

Fixed by not allowing the ROUND operation to add any more decimal places than
was decided in fix_length_and_dec.

2) fix_length_and_dec would allow the number of decimals to be greater than
the maximium configured value for constant values of D. This led to the same 
crash as in (1).

Fixed by not allowing the above in fix_length_and_dec.

mysql-test/r/type_decimal.result

@@ -822,4 +822,71 @@ this must not produce error 1048:
 select * from t1 where ua_invited_by_id not in (select ua_id from t1);
 ua_id	ua_invited_by_id
 drop table t1;
+DROP TABLE IF EXISTS t3;
+DROP TABLE IF EXISTS t4;
+CREATE TABLE t1( a NUMERIC, b INT );
+INSERT INTO t1 VALUES (123456, 40), (123456, 40);
+SELECT TRUNCATE( a, b ) AS c FROM t1 ORDER BY c;
+c
+123456
+123456
+SELECT ROUND( a, b ) AS c FROM t1 ORDER BY c;
+c
+123456
+123456
+SELECT ROUND( a, 100 ) AS c FROM t1 ORDER BY c;
+c
+123456.000000000000000000000000000000
+123456.000000000000000000000000000000
+CREATE TABLE t2( a NUMERIC, b INT );
+INSERT INTO t2 VALUES (123456, 100);
+SELECT TRUNCATE( a, b ) AS c FROM t2 ORDER BY c;
+c
+123456
+SELECT ROUND( a, b ) AS c FROM t2 ORDER BY c;
+c
+123456
+CREATE TABLE t3( a DECIMAL, b INT );
+INSERT INTO t3 VALUES (123456, 40), (123456, 40);
+SELECT TRUNCATE( a, b ) AS c FROM t3 ORDER BY c;
+c
+123456
+123456
+SELECT ROUND( a, b ) AS c FROM t3 ORDER BY c;
+c
+123456
+123456
+SELECT ROUND( a, 100 ) AS c FROM t3 ORDER BY c;
+c
+123456.000000000000000000000000000000
+123456.000000000000000000000000000000
+CREATE TABLE t4( a DECIMAL, b INT );
+INSERT INTO t4 VALUES (123456, 40), (123456, 40);
+SELECT TRUNCATE( a, b ) AS c FROM t4 ORDER BY c;
+c
+123456
+123456
+SELECT ROUND( a, b ) AS c FROM t4 ORDER BY c;
+c
+123456
+123456
+SELECT ROUND( a, 100 ) AS c FROM t4 ORDER BY c;
+c
+123456.000000000000000000000000000000
+123456.000000000000000000000000000000
+delete from t1;
+INSERT INTO t1 VALUES (1234567890, 20), (999.99, 5);
+Warnings:
+Note	1265	Data truncated for column 'a' at row 2
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` decimal(10,0) default NULL,
+  `b` int(11) default NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+select round(a,b) as c from t1 order by c;
+c
+1000
+1234567890
+DROP TABLE t1, t2, t3, t4;
 End of 5.0 tests

mysql-test/t/type_decimal.test

@@ -440,5 +440,42 @@ select * from t1 where ua_invited_by_id not in (select ua_id from t1);
 
 drop table t1;
 
---echo End of 5.0 tests
+#
+# Bug #30889: filesort and order by with float/numeric crashes server
+#
+--disable_warnings
+DROP TABLE IF EXISTS t3;
+DROP TABLE IF EXISTS t4;
+--enable_warnings
+CREATE TABLE t1( a NUMERIC, b INT );
+INSERT INTO t1 VALUES (123456, 40), (123456, 40);
+SELECT TRUNCATE( a, b ) AS c FROM t1 ORDER BY c;
+SELECT ROUND( a, b ) AS c FROM t1 ORDER BY c;
+SELECT ROUND( a, 100 ) AS c FROM t1 ORDER BY c;
+
+CREATE TABLE t2( a NUMERIC, b INT );
+INSERT INTO t2 VALUES (123456, 100);
+SELECT TRUNCATE( a, b ) AS c FROM t2 ORDER BY c;
+SELECT ROUND( a, b ) AS c FROM t2 ORDER BY c;
+
+CREATE TABLE t3( a DECIMAL, b INT );
+INSERT INTO t3 VALUES (123456, 40), (123456, 40);
+SELECT TRUNCATE( a, b ) AS c FROM t3 ORDER BY c;
+SELECT ROUND( a, b ) AS c FROM t3 ORDER BY c;
+SELECT ROUND( a, 100 ) AS c FROM t3 ORDER BY c;
+
+CREATE TABLE t4( a DECIMAL, b INT );
+INSERT INTO t4 VALUES (123456, 40), (123456, 40);
+SELECT TRUNCATE( a, b ) AS c FROM t4 ORDER BY c;
+SELECT ROUND( a, b ) AS c FROM t4 ORDER BY c;
+SELECT ROUND( a, 100 ) AS c FROM t4 ORDER BY c;
 
+delete from t1;
+INSERT INTO t1 VALUES (1234567890, 20), (999.99, 5);
+show create table t1;
+
+select round(a,b) as c from t1 order by c;
+
+DROP TABLE t1, t2, t3, t4;
+
+--echo End of 5.0 tests

sql/item_func.cc

@@ -2003,6 +2003,7 @@ void Item_func_round::fix_length_and_dec()
   case DECIMAL_RESULT:
   {
     hybrid_type= DECIMAL_RESULT;
+    decimals_to_set= min(DECIMAL_MAX_SCALE, decimals_to_set);
     int decimals_delta= args[0]->decimals - decimals_to_set;
     int precision= args[0]->decimal_precision();
     int length_increase= ((decimals_delta <= 0) || truncate) ? 0:1;
@@ -2109,7 +2110,7 @@ my_decimal *Item_func_round::decimal_op(my_decimal *decimal_value)
   longlong dec= args[1]->val_int();
   if (dec > 0 || (dec < 0 && args[1]->unsigned_flag))
   {
-    dec= min((ulonglong) dec, DECIMAL_MAX_SCALE);
+    dec= min((ulonglong) dec, decimals);
     decimals= (uint8) dec; // to get correct output
   }
   else if (dec < INT_MIN)

sql/item_func.h

@@ -236,9 +236,40 @@ class Item_func_numhybrid: public Item_func
   my_decimal *val_decimal(my_decimal *);
   String *val_str(String*str);
 
+  /**
+     @brief Performs the operation that this functions implements when the
+     result type is INT.
+
+     @return The result of the operation.
+  */
   virtual longlong int_op()= 0;
+
+  /**
+     @brief Performs the operation that this functions implements when the
+     result type is REAL.
+
+     @return The result of the operation.
+  */
   virtual double real_op()= 0;
+
+  /**
+     @brief Performs the operation that this functions implements when the
+     result type is DECIMAL.
+
+     @param A pointer where the DECIMAL value will be allocated.
+     @return 
+       - 0 If the result is NULL
+       - The same pointer it was given, with the area initialized to the
+         result of the operation.
+  */
   virtual my_decimal *decimal_op(my_decimal *)= 0;
+
+  /**
+     @brief Performs the operation that this functions implements when the
+     result type is a string type.
+
+     @return The result of the operation.
+  */
   virtual String *str_op(String *)= 0;
   bool is_null() { update_null_value(); return null_value; }
 };