MDEV-21977 main.func_math fails due to undefined behaviour

The problem happened in these line:

uval0= (ulonglong) (val0_negative ? -val0 : val0);
uval1= (ulonglong) (val1_negative ? -val1 : val1);

return check_integer_overflow(val0_negative ? -(longlong) res : res,
                              !val0_negative);

when unary minus was performed on -9223372036854775808.
This behavior is undefined in C/C++.

mysql-test/r/func_math.result

@@ -946,5 +946,53 @@ STDDEV_SAMP(a)
 NULL
 DROP TABLE t1;
 #
+# MDEV-21977 main.func_math fails due to undefined behaviour
+#
+SELECT  9223372036854775808 DIV  1;
+9223372036854775808 DIV  1
+9223372036854775808
+SELECT  9223372036854775808 DIV -1;
+ERROR 22003: BIGINT UNSIGNED value is out of range in '(9223372036854775808 DIV -(1))'
+SELECT -9223372036854775808 DIV  1;
+ERROR 22003: BIGINT value is out of range in '(-(9223372036854775808) DIV 1)'
+SELECT -9223372036854775808 DIV -1;
+ERROR 22003: BIGINT value is out of range in '(-(9223372036854775808) DIV -(1))'
+SELECT  9223372036854775808 MOD  1;
+9223372036854775808 MOD  1
+0
+SELECT  9223372036854775808 MOD -1;
+9223372036854775808 MOD -1
+0
+SELECT -9223372036854775808 MOD  1;
+-9223372036854775808 MOD  1
+0
+SELECT -9223372036854775808 MOD -1;
+-9223372036854775808 MOD -1
+0
+SELECT  1 MOD  9223372036854775808;
+1 MOD  9223372036854775808
+1
+SELECT -1 MOD  9223372036854775808;
+-1 MOD  9223372036854775808
+-1
+SELECT  1 MOD -9223372036854775808;
+1 MOD -9223372036854775808
+1
+SELECT -1 MOD -9223372036854775808;
+-1 MOD -9223372036854775808
+-1
+SELECT  9223372036854775808 MOD  9223372036854775808;
+9223372036854775808 MOD  9223372036854775808
+0
+SELECT  9223372036854775808 MOD -9223372036854775808;
+9223372036854775808 MOD -9223372036854775808
+0
+SELECT -9223372036854775808 MOD  9223372036854775808;
+-9223372036854775808 MOD  9223372036854775808
+0
+SELECT -9223372036854775808 MOD -9223372036854775808;
+-9223372036854775808 MOD -9223372036854775808
+0
+#
 # End of 10.1 tests
 #

mysql-test/t/func_math.test

@@ -692,6 +692,35 @@ SELECT STDDEV_SAMP(a) FROM t1;
 DROP TABLE t1;
 
 
+--echo #
+--echo # MDEV-21977 main.func_math fails due to undefined behaviour
+--echo #
+
+SELECT  9223372036854775808 DIV  1;
+--error ER_DATA_OUT_OF_RANGE
+SELECT  9223372036854775808 DIV -1;
+--error ER_DATA_OUT_OF_RANGE
+SELECT -9223372036854775808 DIV  1;
+--error ER_DATA_OUT_OF_RANGE
+SELECT -9223372036854775808 DIV -1;
+
+SELECT  9223372036854775808 MOD  1;
+SELECT  9223372036854775808 MOD -1;
+SELECT -9223372036854775808 MOD  1;
+SELECT -9223372036854775808 MOD -1;
+
+SELECT  1 MOD  9223372036854775808;
+SELECT -1 MOD  9223372036854775808;
+SELECT  1 MOD -9223372036854775808;
+SELECT -1 MOD -9223372036854775808;
+
+SELECT  9223372036854775808 MOD  9223372036854775808;
+SELECT  9223372036854775808 MOD -9223372036854775808;
+SELECT -9223372036854775808 MOD  9223372036854775808;
+SELECT -9223372036854775808 MOD -9223372036854775808;
+
+
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

sql/item_func.cc

@@ -1834,11 +1834,9 @@ longlong Item_func_int_div::val_int()
       raise_integer_overflow();
     return res;
   }
-  
-  longlong val0=args[0]->val_int();
-  longlong val1=args[1]->val_int();
-  bool val0_negative, val1_negative, res_negative;
-  ulonglong uval0, uval1, res;
+
+  Longlong_hybrid val0= args[0]->to_longlong_hybrid();
+  Longlong_hybrid val1= args[1]->to_longlong_hybrid();
   if ((null_value= (args[0]->null_value || args[1]->null_value)))
     return 0;
   if (val1 == 0)
@@ -1847,12 +1845,8 @@ longlong Item_func_int_div::val_int()
     return 0;
   }
 
-  val0_negative= !args[0]->unsigned_flag && val0 < 0;
-  val1_negative= !args[1]->unsigned_flag && val1 < 0;
-  res_negative= val0_negative != val1_negative;
-  uval0= (ulonglong) (val0_negative ? -val0 : val0);
-  uval1= (ulonglong) (val1_negative ? -val1 : val1);
-  res= uval0 / uval1;
+  bool res_negative= val0.neg() != val1.neg();
+  ulonglong res= val0.abs() / val1.abs();
   if (res_negative)
   {
     if (res > (ulonglong) LONGLONG_MAX)
@@ -1880,11 +1874,8 @@ void Item_func_int_div::fix_length_and_dec()
 longlong Item_func_mod::int_op()
 {
   DBUG_ASSERT(fixed == 1);
-  longlong val0= args[0]->val_int();
-  longlong val1= args[1]->val_int();
-  bool val0_negative, val1_negative;
-  ulonglong uval0, uval1;
-  ulonglong res;
+  Longlong_hybrid val0= args[0]->to_longlong_hybrid();
+  Longlong_hybrid val1= args[1]->to_longlong_hybrid();
 
   if ((null_value= args[0]->null_value || args[1]->null_value))
     return 0; /* purecov: inspected */
@@ -1899,13 +1890,9 @@ longlong Item_func_mod::int_op()
     LONGLONG_MIN by -1 generates SIGFPE, we calculate using unsigned values and
     then adjust the sign appropriately.
   */
-  val0_negative= !args[0]->unsigned_flag && val0 < 0;
-  val1_negative= !args[1]->unsigned_flag && val1 < 0;
-  uval0= (ulonglong) (val0_negative ? -val0 : val0);
-  uval1= (ulonglong) (val1_negative ? -val1 : val1);
-  res= uval0 % uval1;
-  return check_integer_overflow(val0_negative ? -(longlong) res : res,
-                                !val0_negative);
+  ulonglong res= val0.abs() % val1.abs();
+  return check_integer_overflow(val0.neg() ? -(longlong) res : res,
+                                !val0.neg());
 }
 
 double Item_func_mod::real_op()

sql/sql_type_int.h

@@ -67,6 +67,26 @@ class Longlong_hybrid
     */
     return cmp_signed(other);
   }
+  bool operator==(const Longlong_hybrid &nr) const
+  {
+    return cmp(nr) == 0;
+  }
+  bool operator==(ulonglong nr) const
+  {
+    return cmp(Longlong_hybrid((longlong) nr, true)) == 0;
+  }
+  bool operator==(uint nr) const
+  {
+    return cmp(Longlong_hybrid((longlong) nr, true)) == 0;
+  }
+  bool operator==(longlong nr) const
+  {
+    return cmp(Longlong_hybrid(nr, false)) == 0;
+  }
+  bool operator==(int nr) const
+  {
+    return cmp(Longlong_hybrid(nr, false)) == 0;
+  }
 };
 
 #endif // SQL_TYPE_INT_INCLUDED