MDEV-23323 Rounding functions return a wrong data type for a BIT, ENUM, SET argument

Implementing dedicated fixing methods:
- Type_handler_bit::Item_func_round_fix_length_and_dec()
- Type_handler_bit::Item_func_int_val_fix_length_and_dec()
- Type_handler_typelib::Item_func_round_fix_length_and_dec()

because the inherited methods did not work well.

Fixing:
- Type_handler_typelib::Item_func_int_val_fix_length_and_dec
  It did not work well, because it used args[0]->max_length to
  calculate the result data type. In case of ENUM and SET it was
  not correct, because in FLOOR() and CEILING() context
  ENUM and SET return not more than 5 digits (65535 is the biggest
  possible value).

Misc:
- Changing the API of
    Type_handler_bit::Bit_decimal_notation_int_digits(const Item *item)
  to a more generic form:
    Type_handler_bit::Bit_decimal_notation_int_digits_by_nbits(uint nbits)

- Fixing Type_handler_bit::Bit_decimal_notation_int_digits_by_nbits() to
  return the exact number of decimal digits for all nbits 1..64.
  The old implementation was approximate.
  This change gives better (more precise) data types.

mysql-test/main/type_bit.test

@@ -507,6 +507,35 @@ EXPLAIN SELECT * FROM t1 WHERE a=200;
 EXPLAIN SELECT * FROM t1 WHERE a<=>200;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-23323 Rounding functions return a wrong data type for a BIT, ENUM, SET argument
+--echo #
+
+--vertical_results
+DELIMITER $$;
+BEGIN NOT ATOMIC
+  FOR i IN 1..64
+  DO
+    SELECT '-----', CONCAT('BIT(',i,')') AS Type;
+    EXECUTE IMMEDIATE REPLACE('CREATE TABLE t1 (a BIT(64))','64', i);
+    INSERT IGNORE INTO t1 VALUES (0xFFFFFFFFFFFFFFFF);
+     CREATE TABLE t2 AS SELECT
+       a,
+       FLOOR(a) AS cf,
+       CEILING(a) AS cc,
+       ROUND(a) AS cr,
+       TRUNCATE(a,0) AS ct
+     FROM t1;
+    SHOW CREATE TABLE t2;
+    SELECT CAST(a AS UNSIGNED) AS a, cf, cc, cr, ct FROM t2;
+    DROP TABLE t2;
+    DROP TABLE t1;
+  END FOR;
+END;
+$$
+DELIMITER ;$$
+--horizontal_results
+
 
 --echo #
 --echo # End of 10.4 tests

mysql-test/main/type_enum.result

@@ -2311,3 +2311,24 @@ t2	CREATE TABLE `t2` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP PROCEDURE p1;
 DROP TABLE t1;
+#
+# MDEV-23323 Rounding functions return a wrong data type for a BIT, ENUM, SET argument
+#
+CREATE TABLE t1 (a ENUM('999999999999999999999999999999999999999999999999999999999999'));
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 AS
+SELECT a, FLOOR(a), CEILING(a), TRUNCATE(a,0), ROUND(a) FROM t1;
+SHOW CREATE TABLE t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` enum('999999999999999999999999999999999999999999999999999999999999') DEFAULT NULL,
+  `FLOOR(a)` int(5) unsigned DEFAULT NULL,
+  `CEILING(a)` int(5) unsigned DEFAULT NULL,
+  `TRUNCATE(a,0)` int(5) unsigned DEFAULT NULL,
+  `ROUND(a)` int(5) unsigned DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+SELECT * FROM t2;
+a	FLOOR(a)	CEILING(a)	TRUNCATE(a,0)	ROUND(a)
+999999999999999999999999999999999999999999999999999999999999	1	1	1	1
+DROP TABLE t2;
+DROP TABLE t1;

mysql-test/main/type_enum.test

@@ -518,3 +518,16 @@ DELIMITER ;$$
 CALL p1();
 DROP PROCEDURE p1;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-23323 Rounding functions return a wrong data type for a BIT, ENUM, SET argument
+--echo #
+
+CREATE TABLE t1 (a ENUM('999999999999999999999999999999999999999999999999999999999999'));
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 AS
+  SELECT a, FLOOR(a), CEILING(a), TRUNCATE(a,0), ROUND(a) FROM t1;
+SHOW CREATE TABLE t2;
+SELECT * FROM t2;
+DROP TABLE t2;
+DROP TABLE t1;

mysql-test/main/type_set.result

@@ -358,3 +358,24 @@ DROP TABLE t1;
 SET NAMES utf8;
 CREATE TABLE t1 (a SET('a,bü'));
 ERROR 22007: Illegal set 'a,bü' value found during parsing
+#
+# MDEV-23323 Rounding functions return a wrong data type for a BIT, ENUM, SET argument
+#
+CREATE TABLE t1 (a SET('999999999999999999999999999999999999999999999999999999999999'));
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 AS
+SELECT a, FLOOR(a), CEILING(a), TRUNCATE(a,0), ROUND(a) FROM t1;
+SHOW CREATE TABLE t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` set('999999999999999999999999999999999999999999999999999999999999') DEFAULT NULL,
+  `FLOOR(a)` int(5) unsigned DEFAULT NULL,
+  `CEILING(a)` int(5) unsigned DEFAULT NULL,
+  `TRUNCATE(a,0)` int(5) unsigned DEFAULT NULL,
+  `ROUND(a)` int(5) unsigned DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+SELECT * FROM t2;
+a	FLOOR(a)	CEILING(a)	TRUNCATE(a,0)	ROUND(a)
+999999999999999999999999999999999999999999999999999999999999	1	1	1	1
+DROP TABLE t2;
+DROP TABLE t1;

mysql-test/main/type_set.test

@@ -248,3 +248,16 @@ DROP TABLE t1;
 SET NAMES utf8;
 --error ER_ILLEGAL_VALUE_FOR_TYPE
 CREATE TABLE t1 (a SET('a,bü'));
+
+--echo #
+--echo # MDEV-23323 Rounding functions return a wrong data type for a BIT, ENUM, SET argument
+--echo #
+
+CREATE TABLE t1 (a SET('999999999999999999999999999999999999999999999999999999999999'));
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 AS
+  SELECT a, FLOOR(a), CEILING(a), TRUNCATE(a,0), ROUND(a) FROM t1;
+SHOW CREATE TABLE t2;
+SELECT * FROM t2;
+DROP TABLE t2;
+DROP TABLE t1;

sql/item_func.cc

@@ -2218,9 +2218,13 @@ bool Item_func_int_val::fix_length_and_dec()
 {
   DBUG_ENTER("Item_func_int_val::fix_length_and_dec");
   DBUG_PRINT("info", ("name %s", func_name()));
-  if (args[0]->type_handler()->Item_func_int_val_fix_length_and_dec(this))
+  /*
+    We don't want to translate ENUM/SET to CHAR here.
+    So let's call real_type_handler(), not type_handler().
+  */
+  if (args[0]->real_type_handler()->Item_func_int_val_fix_length_and_dec(this))
     DBUG_RETURN(TRUE);
-  DBUG_PRINT("info", ("Type: %s", type_handler()->name().ptr()));
+  DBUG_PRINT("info", ("Type: %s", real_type_handler()->name().ptr()));
   DBUG_RETURN(FALSE);
 }
 

sql/item_func.h

@@ -458,6 +458,29 @@ class Item_hybrid_func: public Item_func,
   {
     Type_geometry_attributes::set_geometry_type(type);
   }
+  void fix_length_and_dec_long_or_longlong(uint char_length, bool unsigned_arg)
+  {
+    collation.set_numeric();
+    unsigned_flag= unsigned_arg;
+    max_length= char_length;
+#if MARIADB_VERSION_ID < 100500
+    set_handler(Type_handler::type_handler_long_or_longlong(char_length));
+#else
+    set_handler(Type_handler::type_handler_long_or_longlong(char_length,
+                                                            unsigned_arg));
+#endif
+  }
+  void fix_length_and_dec_ulong_or_ulonglong_by_nbits(uint nbits)
+  {
+    uint digits= Type_handler_bit::Bit_decimal_notation_int_digits_by_nbits(nbits);
+    collation.set_numeric();
+    unsigned_flag= true;
+    max_length= digits;
+    if (nbits > 32)
+      set_handler(&type_handler_longlong);
+    else
+      set_handler(&type_handler_long);
+  }
 };
 
 
@@ -1758,7 +1781,11 @@ class Item_func_round :public Item_func_hybrid_field_type
   void fix_arg_temporal(const Type_handler *h, uint int_part_length);
   bool fix_length_and_dec()
   {
-    return args[0]->type_handler()->Item_func_round_fix_length_and_dec(this);
+    /*
+      We don't want to translate ENUM/SET to CHAR here.
+      So let's real_type_handler(), not type_handler().
+    */
+    return args[0]->real_type_handler()->Item_func_round_fix_length_and_dec(this);
   }
   Item *get_copy(THD *thd)
   { return get_item_copy<Item_func_round>(thd, this); }

sql/sql_type.cc

@@ -3536,7 +3536,7 @@ uint32
 Type_handler_bit::Item_decimal_notation_int_digits(const Item *item)
                                                               const
 {
-  return Bit_decimal_notation_int_digits(item);
+  return Bit_decimal_notation_int_digits_by_nbits(item->max_length);
 }
 
 
@@ -3554,9 +3554,23 @@ Type_handler_general_purpose_int::Item_decimal_notation_int_digits(
     a divisor.
 */
 uint32
-Type_handler_bit::Bit_decimal_notation_int_digits(const Item *item)
-{
-  return item->max_length/3+1;
+Type_handler_bit::Bit_decimal_notation_int_digits_by_nbits(uint nbits)
+{
+  DBUG_ASSERT(nbits > 0);
+  DBUG_ASSERT(nbits <= 64);
+  set_if_smaller(nbits, 64); // Safety
+  static uint ndigits[65]=
+  {0,
+   1,1,1,2,2,2,3,3,         // 1..8 bits
+   3,4,4,4,4,5,5,5,         // 9..16 bits
+   6,6,6,7,7,7,7,8,         // 17..24 bits
+   8,8,9,9,9,10,10,10,      // 25..32 bits
+   10,11,11,11,12,12,12,13, // 33..40 bits
+   13,13,13,14,14,14,15,15, // 41..48 bits
+   15,16,16,16,16,17,17,17, // 49..56 bits
+   18,18,18,19,19,19,19,20  // 57..64 bits
+  };
+  return ndigits[nbits];
 }
 
 /*************************************************************************/
@@ -5658,6 +5672,23 @@ bool Type_handler_hex_hybrid::
 }
 
 
+bool Type_handler_bit::
+       Item_func_round_fix_length_and_dec(Item_func_round *item) const
+{
+  uint nbits= item->arguments()[0]->max_length;
+  item->fix_length_and_dec_ulong_or_ulonglong_by_nbits(nbits);
+  return false;
+}
+
+
+bool Type_handler_typelib::
+       Item_func_round_fix_length_and_dec(Item_func_round *item) const
+{
+  item->fix_length_and_dec_long_or_longlong(5, true);
+  return false;
+}
+
+
 bool Type_handler_real_result::
        Item_func_round_fix_length_and_dec(Item_func_round *item) const
 {
@@ -5740,10 +5771,19 @@ bool Type_handler_int_result::
 }
 
 
+bool Type_handler_bit::
+       Item_func_int_val_fix_length_and_dec(Item_func_int_val *item) const
+{
+  uint nbits= item->arguments()[0]->max_length;
+  item->fix_length_and_dec_ulong_or_ulonglong_by_nbits(nbits);
+  return false;
+}
+
+
 bool Type_handler_typelib::
        Item_func_int_val_fix_length_and_dec(Item_func_int_val *item) const
 {
-  item->fix_length_and_dec_int_or_decimal();
+  item->fix_length_and_dec_long_or_longlong(5, true);
   return false;
 }
 
@@ -5751,14 +5791,8 @@ bool Type_handler_typelib::
 bool Type_handler_hex_hybrid::
        Item_func_int_val_fix_length_and_dec(Item_func_int_val *item) const
 {
-  item->collation.set_numeric();
-  item->unsigned_flag= true;
-  item->max_length= item->arguments()[0]->decimal_precision();
-#if MARIADB_VERSION_ID < 100500
-  item->set_handler(type_handler_long_or_longlong(item->max_length));
-#else
-  item->set_handler(type_handler_long_or_longlong(item->max_length, true));
-#endif
+  uint nchars= item->arguments()[0]->decimal_precision();
+  item->fix_length_and_dec_long_or_longlong(nchars, true);
   return false;
 }
 

sql/sql_type.h

@@ -5147,7 +5147,7 @@ class Type_handler_bit: public Type_handler_int_result
   }
   uint32 max_display_length(const Item *item) const;
   uint32 Item_decimal_notation_int_digits(const Item *item) const;
-  static uint32 Bit_decimal_notation_int_digits(const Item *item); 
+  static uint32 Bit_decimal_notation_int_digits_by_nbits(uint nbits);
   uint32 calc_pack_length(uint32 length) const { return length / 8; }
   bool Item_send(Item *item, Protocol *protocol, st_value *buf) const
   {
@@ -5157,6 +5157,8 @@ class Type_handler_bit: public Type_handler_int_result
   {
     return print_item_value_csstr(thd, item, str);
   }
+  bool Item_func_round_fix_length_and_dec(Item_func_round *) const;
+  bool Item_func_int_val_fix_length_and_dec(Item_func_int_val *) const;
   Field *make_conversion_table_field(TABLE *, uint metadata,
                                      const Field *target) const;
   bool Column_definition_fix_attributes(Column_definition *c) const;
@@ -6271,6 +6273,7 @@ class Type_handler_typelib: public Type_handler_general_purpose_string
   enum_field_types field_type() const { return MYSQL_TYPE_STRING; }
   const Type_handler *type_handler_for_item_field() const;
   const Type_handler *cast_to_int_type_handler() const;
+  bool Item_func_round_fix_length_and_dec(Item_func_round *) const;
   bool Item_func_int_val_fix_length_and_dec(Item_func_int_val *) const;
   bool Item_hybrid_func_fix_attributes(THD *thd,
                                        const char *name,