Fix for bug #13573 (wrong data inserted for too big decimals)

mysql-test/r/type_newdecimal.result:
  result fixed
mysql-test/t/type_newdecimal.test:
  test case added
sql/item_func.cc:
  conditions fixed
sql/my_decimal.cc:
  overflow handling added
sql/my_decimal.h:
  overflow handling added - so the result of operation gets maximum possible
  decimal value

mysql-test/r/type_newdecimal.result

@@ -846,15 +846,14 @@ select 0/0;
 NULL
 select 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 as x;
 x
-999999999999999999999999999999999999999999999999999999999999999999999999999999999
+99999999999999999999999999999999999999999999999999999999999999999
 Warnings:
 Error	1292	Truncated incorrect DECIMAL value: ''
 select 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 + 1 as x;
 x
-NULL
+100000000000000000000000000000000000000000000000000000000000000000
 Warnings:
 Error	1292	Truncated incorrect DECIMAL value: ''
-Error	1292	Truncated incorrect DECIMAL value: ''
 select 0.190287977636363637 + 0.040372670 * 0 -  0;
 0.190287977636363637 + 0.040372670 * 0 -  0
 0.190287977636363637
@@ -1019,3 +1018,26 @@ drop procedure wg2;
 select cast(@non_existing_user_var/2 as DECIMAL);
 cast(@non_existing_user_var/2 as DECIMAL)
 NULL
+create table t1 (c1 decimal(64));
+insert into t1 values(
+89000000000000000000000000000000000000000000000000000000000000000000000000000000000000000);
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Warning	1264	Out of range value adjusted for column 'c1' at row 1
+insert into t1 values(
+99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 *
+99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999);
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
+Warning	1264	Out of range value adjusted for column 'c1' at row 1
+insert into t1 values(1e100);
+Warnings:
+Warning	1264	Out of range value adjusted for column 'c1' at row 1
+select * from t1;
+c1
+9999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999
+drop table t1;

mysql-test/t/type_newdecimal.test

@@ -1044,3 +1044,18 @@ drop procedure wg2;
 #
 
 select cast(@non_existing_user_var/2 as DECIMAL);
+
+
+#
+# Bug #13573 (Wrong data inserted for too big values)
+#
+
+create table t1 (c1 decimal(64));
+insert into t1 values(
+89000000000000000000000000000000000000000000000000000000000000000000000000000000000000000);
+insert into t1 values(
+99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 *
+99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999);
+insert into t1 values(1e100);
+select * from t1;
+drop table t1;

sql/item_func.cc

@@ -972,8 +972,8 @@ my_decimal *Item_func_plus::decimal_op(my_decimal *decimal_value)
     return 0;
   val2= args[1]->val_decimal(&value2);
   if (!(null_value= (args[1]->null_value ||
-                     my_decimal_add(E_DEC_FATAL_ERROR, decimal_value, val1,
-                                    val2) > 1)))
+                     (my_decimal_add(E_DEC_FATAL_ERROR, decimal_value, val1,
+                                     val2) > 3))))
     return decimal_value;
   return 0;
 }
@@ -1045,8 +1045,8 @@ my_decimal *Item_func_minus::decimal_op(my_decimal *decimal_value)
     return 0;
   val2= args[1]->val_decimal(&value2);
   if (!(null_value= (args[1]->null_value ||
-                     my_decimal_sub(E_DEC_FATAL_ERROR, decimal_value, val1,
-                                    val2) > 1)))
+                     (my_decimal_sub(E_DEC_FATAL_ERROR, decimal_value, val1,
+                                     val2) > 3))))
     return decimal_value;
   return 0;
 }
@@ -1083,8 +1083,8 @@ my_decimal *Item_func_mul::decimal_op(my_decimal *decimal_value)
     return 0;
   val2= args[1]->val_decimal(&value2);
   if (!(null_value= (args[1]->null_value ||
-                     my_decimal_mul(E_DEC_FATAL_ERROR, decimal_value, val1,
-                                    val2) > 1)))
+                     (my_decimal_mul(E_DEC_FATAL_ERROR, decimal_value, val1,
+                                    val2) > 3))))
     return decimal_value;
   return 0;
 }

sql/my_decimal.cc

@@ -185,7 +185,7 @@ int str2my_decimal(uint mask, const char *from, uint length,
       }
     }
   }
-  check_result(mask, err);
+  check_result_and_overflow(mask, err, decimal_value);
   return err;
 }
 

sql/my_decimal.h

@@ -126,6 +126,19 @@ inline int decimal_operation_results(int result)
 }
 #endif /*MYSQL_CLIENT*/
 
+inline
+void max_my_decimal(my_decimal *to, int precision, int frac)
+{
+  DBUG_ASSERT((precision <= DECIMAL_MAX_PRECISION)&&
+              (frac <= DECIMAL_MAX_SCALE));
+  max_decimal(precision, frac, (decimal_t*) to);
+}
+
+inline void max_internal_decimal(my_decimal *to)
+{
+  max_my_decimal(to, DECIMAL_MAX_PRECISION, 0);
+}
+
 inline int check_result(uint mask, int result)
 {
   if (result & mask)
@@ -133,6 +146,18 @@ inline int check_result(uint mask, int result)
   return result;
 }
 
+inline int check_result_and_overflow(uint mask, int result, my_decimal *val)
+{
+  if (check_result(mask, result) & E_DEC_OVERFLOW)
+  {
+    bool sign= val->sign();
+    val->fix_buffer_pointer();
+    max_internal_decimal(val);
+    val->sign(sign);
+  }
+  return result;
+}
+
 inline uint my_decimal_length_to_precision(uint length, uint scale,
                                            bool unsigned_flag)
 {
@@ -256,7 +281,8 @@ int my_decimal2double(uint mask, const my_decimal *d, double *result)
 inline
 int str2my_decimal(uint mask, const char *str, my_decimal *d, char **end)
 {
-  return check_result(mask, string2decimal(str, (decimal_t*) d, end));
+  return check_result_and_overflow(mask, string2decimal(str,(decimal_t*)d,end),
+                                   d);
 }
 
 
@@ -274,7 +300,7 @@ int string2my_decimal(uint mask, const String *str, my_decimal *d)
 inline
 int double2my_decimal(uint mask, double val, my_decimal *d)
 {
-  return check_result(mask, double2decimal(val, (decimal_t*) d));
+  return check_result_and_overflow(mask, double2decimal(val, (decimal_t*)d), d);
 }
 
 
@@ -303,7 +329,9 @@ inline
 int my_decimal_add(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
-  return check_result(mask, decimal_add((decimal_t*) a, (decimal_t*) b, res));
+  return check_result_and_overflow(mask,
+                                   decimal_add((decimal_t*)a,(decimal_t*)b,res),
+                                   res);
 }
 
 
@@ -311,7 +339,9 @@ inline
 int my_decimal_sub(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
-  return check_result(mask, decimal_sub((decimal_t*) a, (decimal_t*) b, res));
+  return check_result_and_overflow(mask,
+                                   decimal_sub((decimal_t*)a,(decimal_t*)b,res),
+                                   res);
 }
 
 
@@ -319,7 +349,9 @@ inline
 int my_decimal_mul(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
-  return check_result(mask, decimal_mul((decimal_t*) a, (decimal_t*) b, res));
+  return check_result_and_overflow(mask,
+                                   decimal_mul((decimal_t*)a,(decimal_t*)b,res),
+                                   res);
 }
 
 
@@ -327,8 +359,10 @@ inline
 int my_decimal_div(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b, int div_scale_inc)
 {
-  return check_result(mask, decimal_div((decimal_t*) a, (decimal_t*) b, res,
-					div_scale_inc));
+  return check_result_and_overflow(mask,
+                                   decimal_div((decimal_t*)a,(decimal_t*)b,res,
+                                               div_scale_inc),
+                                   res);
 }
 
 
@@ -336,7 +370,9 @@ inline
 int my_decimal_mod(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
-  return check_result(mask, decimal_mod((decimal_t*) a, (decimal_t*) b, res));
+  return check_result_and_overflow(mask,
+                                   decimal_mod((decimal_t*)a,(decimal_t*)b,res),
+                                   res);
 }
 
 
@@ -347,13 +383,5 @@ int my_decimal_cmp(const my_decimal *a, const my_decimal *b)
   return decimal_cmp((decimal_t*) a, (decimal_t*) b);
 }
 
-inline
-void max_my_decimal(my_decimal *to, int precision, int frac)
-{
-  DBUG_ASSERT((precision <= DECIMAL_MAX_PRECISION)&&
-              (frac <= DECIMAL_MAX_SCALE));
-  max_decimal(precision, frac, (decimal_t*) to);
-}
-
 #endif /*my_decimal_h*/