crash in string-to-int conversion

using a specially crafted strings one could overflow `shift`
variable and cause a crash by dereferencing d10[-2147483648]
(on a sufficiently old gcc).

This is a correct fix and a test case for

Bug #29723340: MYSQL SERVER CRASH AFTER SQL QUERY WITH DATA ?AST

mysql-test/r/func_math.result

@@ -829,5 +829,45 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP TABLE t1;
 #
+# Bug #29723340: MYSQL SERVER CRASH AFTER SQL QUERY WITH DATA ?AST
+#
+create table t1(a int);
+insert t1 values("1e-214748364");
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+insert t1 values("1e-2147483648");
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+insert t1 values("1e-21474836480");
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+insert t1 values("1e+214748364");
+Warnings:
+Warning	1264	Out of range value for column 'a' at row 1
+insert t1 values("1e+2147483647");
+Warnings:
+Warning	1264	Out of range value for column 'a' at row 1
+insert t1 values("1e+21474836470");
+Warnings:
+Warning	1264	Out of range value for column 'a' at row 1
+set global max_allowed_packet= cast(2*1024*1024*1024+1024 as unsigned);
+Warnings:
+Warning	1292	Truncated incorrect max_allowed_packet value: '2147484672'
+set @a=2147483647;
+insert t1 values (concat('1', repeat('0', @a+18), 'e-', @a-1, '0'));
+Warnings:
+Warning	1301	Result of repeat() was larger than max_allowed_packet (1073741824) - truncated
+set global max_allowed_packet=default;
+select * from t1;
+a
+0
+0
+0
+2147483647
+2147483647
+2147483647
+NULL
+drop table t1;
+#
 # End of 5.5 tests
 #

mysql-test/t/func_math.test

@@ -614,6 +614,31 @@ SHOW CREATE TABLE t1;
 DROP TABLE t1;
 
 
+--echo #
+--echo # Bug #29723340: MYSQL SERVER CRASH AFTER SQL QUERY WITH DATA ?AST
+--echo #
+
+create table t1(a int);
+insert t1 values("1e-214748364");
+insert t1 values("1e-2147483648");
+insert t1 values("1e-21474836480");
+insert t1 values("1e+214748364");
+insert t1 values("1e+2147483647");
+insert t1 values("1e+21474836470");
+
+# if max max_allowed_packet will ever be increased beyond 2GB, this could
+# break again:
+set global max_allowed_packet= cast(2*1024*1024*1024+1024 as unsigned);
+connect foo,localhost,root;
+set @a=2147483647;
+insert t1 values (concat('1', repeat('0', @a+18), 'e-', @a-1, '0'));
+disconnect foo;
+connection default;
+set global max_allowed_packet=default;
+
+select * from t1;
+drop table t1;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

strings/ctype-simple.c

 /* Copyright (c) 2002, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2014, SkySQL Ab.
+   Copyright (c) 2009, 2019, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1524,10 +1524,20 @@ my_strntoull10rnd_8bit(CHARSET_INFO *cs __attribute__((unused)),
         if (++str == end)
           goto ret_sign;
       }
-      for (exponent= 0 ;
-           str < end && (ch= (uchar) (*str - '0')) < 10;
-           str++)
+      if (shift > 0 && !negative_exp)
+        goto ret_too_big;
+      for (exponent= 0 ; str < end && (ch= (uchar) (*str - '0')) < 10; str++)
       {
+        if (negative_exp)
+        {
+          if (exponent - shift > DIGITS_IN_ULONGLONG)
+            goto ret_zero;
+        }
+        else
+        {
+          if (exponent + shift > DIGITS_IN_ULONGLONG)
+            goto ret_too_big;
+        }
         exponent= exponent * 10 + ch;
       }
       shift+= negative_exp ? -exponent : exponent;