Bug #11792200 - DIVIDING LARGE NUMBERS CAUSES STACK CORRUPTIONS

This was a buffer overrun in do_div_mod(), overwriting the internal buffer
of auto variable 'tmp' in Item_func_int_div::val_int.
Result on windows: 'this' is set to zero, and crash.
Ran fine on other platforms (no valgrind warnings),
but this is undefined behaviour on any platform of course.

include/decimal.h

@@ -29,14 +29,14 @@ typedef struct st_decimal_t {
 
 int internal_str2dec(const char *from, decimal_t *to, char **end,
                      my_bool fixed);
-int decimal2string(decimal_t *from, char *to, int *to_len,
+int decimal2string(const decimal_t *from, char *to, int *to_len,
                    int fixed_precision, int fixed_decimals,
                    char filler);
 int decimal2ulonglong(decimal_t *from, ulonglong *to);
 int ulonglong2decimal(ulonglong from, decimal_t *to);
 int decimal2longlong(decimal_t *from, longlong *to);
 int longlong2decimal(longlong from, decimal_t *to);
-int decimal2double(decimal_t *from, double *to);
+int decimal2double(const decimal_t *from, double *to);
 int double2decimal(double from, decimal_t *to);
 int decimal_actual_fraction(decimal_t *from);
 int decimal2bin(decimal_t *from, uchar *to, int precision, int scale);
@@ -47,17 +47,17 @@ int decimal_bin_size(int precision, int scale);
 int decimal_result_size(decimal_t *from1, decimal_t *from2, char op,
                         int param);
 
-int decimal_intg(decimal_t *from);
-int decimal_add(decimal_t *from1, decimal_t *from2, decimal_t *to);
-int decimal_sub(decimal_t *from1, decimal_t *from2, decimal_t *to);
-int decimal_cmp(decimal_t *from1, decimal_t *from2);
-int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to);
-int decimal_div(decimal_t *from1, decimal_t *from2, decimal_t *to,
+int decimal_intg(const decimal_t *from);
+int decimal_add(const decimal_t *from1, const decimal_t *from2, decimal_t *to);
+int decimal_sub(const decimal_t *from1, const decimal_t *from2, decimal_t *to);
+int decimal_cmp(const decimal_t *from1, const decimal_t *from2);
+int decimal_mul(const decimal_t *from1, const decimal_t *from2, decimal_t *to);
+int decimal_div(const decimal_t *from1, const decimal_t *from2, decimal_t *to,
                 int scale_incr);
-int decimal_mod(decimal_t *from1, decimal_t *from2, decimal_t *to);
-int decimal_round(decimal_t *from, decimal_t *to, int new_scale,
+int decimal_mod(const decimal_t *from1, const decimal_t *from2, decimal_t *to);
+int decimal_round(const decimal_t *from, decimal_t *to, int new_scale,
                   decimal_round_mode mode);
-int decimal_is_zero(decimal_t *from);
+int decimal_is_zero(const decimal_t *from);
 void max_decimal(int precision, int frac, decimal_t *to);
 
 #define string2decimal(A,B,C) internal_str2dec((A), (B), (C), 0)

mysql-test/r/func_math.result

@@ -656,3 +656,11 @@ Warning	1366	Incorrect decimal value: '' for column '' at row -1
 SELECT 1 div null;
 1 div null
 NULL
+#
+# Bug #11792200 - DIVIDING LARGE NUMBERS CAUSES STACK CORRUPTIONS
+#
+select (1.175494351E-37 div 1.7976931348623157E+308);
+(1.175494351E-37 div 1.7976931348623157E+308)
+0
+Warnings:
+Warning	1292	Truncated incorrect DECIMAL value: ''

mysql-test/t/func_math.test

@@ -500,3 +500,8 @@ SELECT ((@a:=@b:=1.0) div (@b:=@a:=get_format(datetime, 'usa')));
 --echo # Bug #59498 div function broken in mysql-trunk
 --echo #
 SELECT 1 div null;
+
+--echo #
+--echo # Bug #11792200 - DIVIDING LARGE NUMBERS CAUSES STACK CORRUPTIONS
+--echo #
+select (1.175494351E-37 div 1.7976931348623157E+308);

sql/my_decimal.h

@@ -178,7 +178,7 @@ void max_my_decimal(my_decimal *to, int precision, int frac)
 {
   DBUG_ASSERT((precision <= DECIMAL_MAX_PRECISION)&&
               (frac <= DECIMAL_MAX_SCALE));
-  max_decimal(precision, frac, (decimal_t*) to);
+  max_decimal(precision, frac, to);
 }
 
 inline void max_internal_decimal(my_decimal *to)
@@ -277,14 +277,19 @@ inline
 int binary2my_decimal(uint mask, const uchar *bin, my_decimal *d, int prec,
 		      int scale)
 {
-  return check_result(mask, bin2decimal(bin, (decimal_t*) d, prec, scale));
+  return check_result(mask, bin2decimal(bin, d, prec, scale));
 }
 
 
 inline
 int my_decimal_set_zero(my_decimal *d)
 {
-  decimal_make_zero(((decimal_t*) d));
+  /*
+    We need the up-cast here, since my_decimal has sign() member functions,
+    which conflicts with decimal_t::size
+    (and decimal_make_zero is a macro, rather than a funcion).
+  */
+  decimal_make_zero(static_cast<decimal_t*>(d));
   return 0;
 }
 
@@ -292,7 +297,7 @@ int my_decimal_set_zero(my_decimal *d)
 inline
 bool my_decimal_is_zero(const my_decimal *decimal_value)
 {
-  return decimal_is_zero((decimal_t*) decimal_value);
+  return decimal_is_zero(decimal_value);
 }
 
 
@@ -300,7 +305,7 @@ inline
 int my_decimal_round(uint mask, const my_decimal *from, int scale,
                      bool truncate, my_decimal *to)
 {
-  return check_result(mask, decimal_round((decimal_t*) from, to, scale,
+  return check_result(mask, decimal_round(from, to, scale,
 					  (truncate ? TRUNCATE : HALF_UP)));
 }
 
@@ -308,14 +313,14 @@ int my_decimal_round(uint mask, const my_decimal *from, int scale,
 inline
 int my_decimal_floor(uint mask, const my_decimal *from, my_decimal *to)
 {
-  return check_result(mask, decimal_round((decimal_t*) from, to, 0, FLOOR));
+  return check_result(mask, decimal_round(from, to, 0, FLOOR));
 }
 
 
 inline
 int my_decimal_ceiling(uint mask, const my_decimal *from, my_decimal *to)
 {
-  return check_result(mask, decimal_round((decimal_t*) from, to, 0, CEILING));
+  return check_result(mask, decimal_round(from, to, 0, CEILING));
 }
 
 
@@ -337,7 +342,7 @@ int my_decimal2int(uint mask, const my_decimal *d, my_bool unsigned_flag,
 {
   my_decimal rounded;
   /* decimal_round can return only E_DEC_TRUNCATED */
-  decimal_round((decimal_t*)d, &rounded, 0, HALF_UP);
+  decimal_round(d, &rounded, 0, HALF_UP);
   return check_result(mask, (unsigned_flag ?
 			     decimal2ulonglong(&rounded, (ulonglong *)l) :
 			     decimal2longlong(&rounded, l)));
@@ -348,15 +353,14 @@ inline
 int my_decimal2double(uint, const my_decimal *d, double *result)
 {
   /* No need to call check_result as this will always succeed */
-  return decimal2double((decimal_t*) d, result);
+  return decimal2double(d, result);
 }
 
 
 inline
 int str2my_decimal(uint mask, const char *str, my_decimal *d, char **end)
 {
-  return check_result_and_overflow(mask, string2decimal(str,(decimal_t*)d,end),
-                                   d);
+  return check_result_and_overflow(mask, string2decimal(str, d, end), d);
 }
 
 
@@ -379,7 +383,7 @@ my_decimal *date2my_decimal(MYSQL_TIME *ltime, my_decimal *dec);
 inline
 int double2my_decimal(uint mask, double val, my_decimal *d)
 {
-  return check_result_and_overflow(mask, double2decimal(val, (decimal_t*)d), d);
+  return check_result_and_overflow(mask, double2decimal(val, d), d);
 }
 
 
@@ -409,7 +413,7 @@ int my_decimal_add(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
   return check_result_and_overflow(mask,
-                                   decimal_add((decimal_t*)a,(decimal_t*)b,res),
+                                   decimal_add(a, b, res),
                                    res);
 }
 
@@ -419,7 +423,7 @@ int my_decimal_sub(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
   return check_result_and_overflow(mask,
-                                   decimal_sub((decimal_t*)a,(decimal_t*)b,res),
+                                   decimal_sub(a, b, res),
                                    res);
 }
 
@@ -429,7 +433,7 @@ int my_decimal_mul(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
   return check_result_and_overflow(mask,
-                                   decimal_mul((decimal_t*)a,(decimal_t*)b,res),
+                                   decimal_mul(a, b, res),
                                    res);
 }
 
@@ -439,8 +443,7 @@ int my_decimal_div(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b, int div_scale_inc)
 {
   return check_result_and_overflow(mask,
-                                   decimal_div((decimal_t*)a,(decimal_t*)b,res,
-                                               div_scale_inc),
+                                   decimal_div(a, b, res, div_scale_inc),
                                    res);
 }
 
@@ -450,7 +453,7 @@ int my_decimal_mod(uint mask, my_decimal *res, const my_decimal *a,
 		   const my_decimal *b)
 {
   return check_result_and_overflow(mask,
-                                   decimal_mod((decimal_t*)a,(decimal_t*)b,res),
+                                   decimal_mod(a, b, res),
                                    res);
 }
 
@@ -462,14 +465,14 @@ int my_decimal_mod(uint mask, my_decimal *res, const my_decimal *a,
 inline
 int my_decimal_cmp(const my_decimal *a, const my_decimal *b)
 {
-  return decimal_cmp((decimal_t*) a, (decimal_t*) b);
+  return decimal_cmp(a, b);
 }
 
 
 inline
 int my_decimal_intg(const my_decimal *a)
 {
-  return decimal_intg((decimal_t*) a);
+  return decimal_intg(a);
 }
 
 

strings/decimal.c

@@ -248,7 +248,7 @@ void max_decimal(int precision, int frac, decimal_t *to)
 }
 
 
-static dec1 *remove_leading_zeroes(decimal_t *from, int *intg_result)
+static dec1 *remove_leading_zeroes(const decimal_t *from, int *intg_result)
 {
   int intg= from->intg, i;
   dec1 *buf0= from->buf;
@@ -326,7 +326,7 @@ int decimal_actual_fraction(decimal_t *from)
     E_DEC_OK/E_DEC_TRUNCATED/E_DEC_OVERFLOW
 */
 
-int decimal2string(decimal_t *from, char *to, int *to_len,
+int decimal2string(const decimal_t *from, char *to, int *to_len,
                    int fixed_precision, int fixed_decimals,
                    char filler)
 {
@@ -942,7 +942,7 @@ internal_str2dec(const char *from, decimal_t *to, char **end, my_bool fixed)
     E_DEC_OK/E_DEC_OVERFLOW/E_DEC_TRUNCATED
 */
 
-int decimal2double(decimal_t *from, double *to)
+int decimal2double(const decimal_t *from, double *to)
 {
   char strbuf[FLOATING_POINT_BUFFER], *end;
   int len= sizeof(strbuf);
@@ -1461,7 +1461,7 @@ int decimal_bin_size(int precision, int scale)
 */
 
 int
-decimal_round(decimal_t *from, decimal_t *to, int scale,
+decimal_round(const decimal_t *from, decimal_t *to, int scale,
               decimal_round_mode mode)
 {
   int frac0=scale>0 ? ROUND_UP(scale) : scale/DIG_PER_DEC1,
@@ -1695,7 +1695,7 @@ int decimal_result_size(decimal_t *from1, decimal_t *from2, char op, int param)
   return -1; /* shut up the warning */
 }
 
-static int do_add(decimal_t *from1, decimal_t *from2, decimal_t *to)
+static int do_add(const decimal_t *from1, const decimal_t *from2, decimal_t *to)
 {
   int intg1=ROUND_UP(from1->intg), intg2=ROUND_UP(from2->intg),
       frac1=ROUND_UP(from1->frac), frac2=ROUND_UP(from2->frac),
@@ -1777,7 +1777,7 @@ static int do_add(decimal_t *from1, decimal_t *from2, decimal_t *to)
 
 /* to=from1-from2.
    if to==0, return -1/0/+1 - the result of the comparison */
-static int do_sub(decimal_t *from1, decimal_t *from2, decimal_t *to)
+static int do_sub(const decimal_t *from1, const decimal_t *from2, decimal_t *to)
 {
   int intg1=ROUND_UP(from1->intg), intg2=ROUND_UP(from2->intg),
       frac1=ROUND_UP(from1->frac), frac2=ROUND_UP(from2->frac);
@@ -1846,7 +1846,7 @@ static int do_sub(decimal_t *from1, decimal_t *from2, decimal_t *to)
   /* ensure that always from1 > from2 (and intg1 >= intg2) */
   if (carry)
   {
-    swap_variables(decimal_t *,from1,from1);
+    swap_variables(const decimal_t *, from1, from2);
     swap_variables(dec1 *,start1, start2);
     swap_variables(int,intg1,intg2);
     swap_variables(int,frac1,frac2);
@@ -1912,35 +1912,35 @@ static int do_sub(decimal_t *from1, decimal_t *from2, decimal_t *to)
   return error;
 }
 
-int decimal_intg(decimal_t *from)
+int decimal_intg(const decimal_t *from)
 {
   int res;
   remove_leading_zeroes(from, &res);
   return res;
 }
 
-int decimal_add(decimal_t *from1, decimal_t *from2, decimal_t *to)
+int decimal_add(const decimal_t *from1, const decimal_t *from2, decimal_t *to)
 {
   if (likely(from1->sign == from2->sign))
     return do_add(from1, from2, to);
   return do_sub(from1, from2, to);
 }
 
-int decimal_sub(decimal_t *from1, decimal_t *from2, decimal_t *to)
+int decimal_sub(const decimal_t *from1, const decimal_t *from2, decimal_t *to)
 {
   if (likely(from1->sign == from2->sign))
     return do_sub(from1, from2, to);
   return do_add(from1, from2, to);
 }
 
-int decimal_cmp(decimal_t *from1, decimal_t *from2)
+int decimal_cmp(const decimal_t *from1, const decimal_t *from2)
 {
   if (likely(from1->sign == from2->sign))
     return do_sub(from1, from2, 0);
   return from1->sign > from2->sign ? -1 : 1;
 }
 
-int decimal_is_zero(decimal_t *from)
+int decimal_is_zero(const decimal_t *from)
 {
   dec1 *buf1=from->buf,
        *end=buf1+ROUND_UP(from->intg)+ROUND_UP(from->frac);
@@ -1971,7 +1971,7 @@ int decimal_is_zero(decimal_t *from)
     XXX if this library is to be used with huge numbers of thousands of
     digits, fast multiplication must be implemented.
 */
-int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to)
+int decimal_mul(const decimal_t *from1, const decimal_t *from2, decimal_t *to)
 {
   int intg1=ROUND_UP(from1->intg), intg2=ROUND_UP(from2->intg),
       frac1=ROUND_UP(from1->frac), frac2=ROUND_UP(from2->frac),
@@ -2095,8 +2095,8 @@ int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to)
   changed to malloc (or at least fallback to malloc if alloca() fails)
   but then, decimal_mul() should be rewritten too :(
 */
-static int do_div_mod(decimal_t *from1, decimal_t *from2,
-                       decimal_t *to, decimal_t *mod, int scale_incr)
+static int do_div_mod(const decimal_t *from1, const decimal_t *from2,
+                      decimal_t *to, decimal_t *mod, int scale_incr)
 {
   int frac1=ROUND_UP(from1->frac)*DIG_PER_DEC1, prec1=from1->intg+frac1,
       frac2=ROUND_UP(from2->frac)*DIG_PER_DEC1, prec2=from2->intg+frac2,
@@ -2181,9 +2181,12 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
   }
   buf0=to->buf;
   stop0=buf0+intg0+frac0;
+  DBUG_ASSERT(stop0 <= &to->buf[to->len]);
   if (likely(div_mod))
-    while (dintg++ < 0)
+    while (dintg++ < 0 && buf0 < &to->buf[to->len])
+    {
       *buf0++=0;
+    }
 
   len1=(i=ROUND_UP(prec1))+ROUND_UP(2*frac2+scale_incr+1) + 1;
   set_if_bigger(len1, 3);
@@ -2355,7 +2358,8 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
 */
 
 int
-decimal_div(decimal_t *from1, decimal_t *from2, decimal_t *to, int scale_incr)
+decimal_div(const decimal_t *from1, const decimal_t *from2, decimal_t *to,
+            int scale_incr)
 {
   return do_div_mod(from1, from2, to, 0, scale_incr);
 }
@@ -2387,7 +2391,7 @@ decimal_div(decimal_t *from1, decimal_t *from2, decimal_t *to, int scale_incr)
    thus, there's no requirement for M or N to be integers
 */
 
-int decimal_mod(decimal_t *from1, decimal_t *from2, decimal_t *to)
+int decimal_mod(const decimal_t *from1, const decimal_t *from2, decimal_t *to)
 {
   return do_div_mod(from1, from2, 0, to, 0);
 }