MDEV-10467 Assertion `nr >= 0.0' failed in Item_sum_std::val_real()

Backporting MDEV-5781 from 10.0.

mysql-test/r/func_math.result

@@ -761,3 +761,15 @@ select 5 div 2.0;
 select 5.9 div 2, 1.23456789e3 DIV 2, 1.23456789e9 DIV 2, 1.23456789e19 DIV 2;
 5.9 div 2	1.23456789e3 DIV 2	1.23456789e9 DIV 2	1.23456789e19 DIV 2
 2	617	617283945	6172839450000000000
+#
+# MDEV-10467 Assertion `nr >= 0.0' failed in Item_sum_std::val_real()
+#
+CREATE TABLE t1 (i INT);
+INSERT INTO t1 VALUES (1),(2);
+SELECT STDDEV_SAMP(ROUND('0', 309)) FROM t1;
+STDDEV_SAMP(ROUND('0', 309))
+0
+DROP TABLE t1;
+#
+# End of 5.5 tests
+#

mysql-test/t/func_math.test

@@ -567,3 +567,15 @@ select 5.0 div 2.0;
 select 5.0 div 2;
 select 5 div 2.0;
 select 5.9 div 2, 1.23456789e3 DIV 2, 1.23456789e9 DIV 2, 1.23456789e19 DIV 2;
+
+--echo #
+--echo # MDEV-10467 Assertion `nr >= 0.0' failed in Item_sum_std::val_real()
+--echo #
+CREATE TABLE t1 (i INT);
+INSERT INTO t1 VALUES (1),(2);
+SELECT STDDEV_SAMP(ROUND('0', 309)) FROM t1;
+DROP TABLE t1;
+
+--echo #
+--echo # End of 5.5 tests
+--echo #

sql/item_func.cc

@@ -2661,6 +2661,9 @@ double my_double_round(double value, longlong dec, bool dec_unsigned,
   volatile double value_div_tmp= value / tmp;
   volatile double value_mul_tmp= value * tmp;
 
+  if (!dec_negative && my_isinf(tmp)) // "dec" is too large positive number
+    return value;
+
   if (dec_negative && my_isinf(tmp))
     tmp2= 0.0;
   else if (!dec_negative && my_isinf(value_mul_tmp))