MDEV-22583: Selectivity for BIT columns in filtered column for EXPLAIN is incorrect

For BIT columns when EITS is collected, we store the integral value in
text representation in the min and max fields of the statistical table
When this value is retrieved from the statistical table to original table
field then we try to store the text representation in the original field
which is INCORRECT.

The value that is retrieved should be converted to integral type and that
value should be stored back in the original field. This would get us the
correct estimate for selectivity of the predicate.

mysql-test/r/selectivity.result

@@ -1887,4 +1887,28 @@ a	b
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1;
 # End of 10.1 tests
+#
+# MDEV-22583: Selectivity for BIT columns in filtered column for EXPLAIN is incorrect
+#
+SET optimizer_use_condition_selectivity=4;
+SET histogram_size=255;
+CREATE TABLE t1 (a BIT(32), b INT);
+INSERT INTO t1 VALUES (80, 80), (81, 81), (82, 82);
+ANALYZE TABLE t1 PERSISTENT FOR ALL;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+EXPLAIN EXTENDED SELECT * from t1 where t1.a >= 81;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	3	66.41	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` >= 81
+SELECT HEX(a), b from t1 where t1.a >= 81;
+HEX(a)	b
+51	81
+52	82
+set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
+set histogram_size=@save_histogram_size;
+DROP TABLE t1;
+# End of 10.2 tests
 set @@global.histogram_size=@save_histogram_size;

mysql-test/r/selectivity_innodb.result

@@ -1897,6 +1897,30 @@ a	b
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1;
 # End of 10.1 tests
+#
+# MDEV-22583: Selectivity for BIT columns in filtered column for EXPLAIN is incorrect
+#
+SET optimizer_use_condition_selectivity=4;
+SET histogram_size=255;
+CREATE TABLE t1 (a BIT(32), b INT);
+INSERT INTO t1 VALUES (80, 80), (81, 81), (82, 82);
+ANALYZE TABLE t1 PERSISTENT FOR ALL;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+EXPLAIN EXTENDED SELECT * from t1 where t1.a >= 81;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	3	66.41	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` >= 81
+SELECT HEX(a), b from t1 where t1.a >= 81;
+HEX(a)	b
+51	81
+52	82
+set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
+set histogram_size=@save_histogram_size;
+DROP TABLE t1;
+# End of 10.2 tests
 set @@global.histogram_size=@save_histogram_size;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 set @tmp_ust= @@use_stat_tables;

mysql-test/t/selectivity.test

@@ -1286,6 +1286,24 @@ drop table t1;
 
 --echo # End of 10.1 tests
 
+--echo #
+--echo # MDEV-22583: Selectivity for BIT columns in filtered column for EXPLAIN is incorrect
+--echo #
+
+SET optimizer_use_condition_selectivity=4;
+SET histogram_size=255;
+CREATE TABLE t1 (a BIT(32), b INT);
+INSERT INTO t1 VALUES (80, 80), (81, 81), (82, 82);
+ANALYZE TABLE t1 PERSISTENT FOR ALL;
+EXPLAIN EXTENDED SELECT * from t1 where t1.a >= 81;
+SELECT HEX(a), b from t1 where t1.a >= 81;
+
+set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
+set histogram_size=@save_histogram_size;
+DROP TABLE t1;
+
+--echo # End of 10.2 tests
+
 #
 # Clean up
 #

sql/sql_statistics.cc

@@ -1154,16 +1154,30 @@ class Column_stat: public Stat_table
 
           switch (i) {
           case COLUMN_STAT_MIN_VALUE:
-	    table_field->read_stats->min_value->set_notnull();
-            stat_field->val_str(&val);
-            table_field->read_stats->min_value->store(val.ptr(), val.length(),
-                                                      &my_charset_bin);
+            table_field->read_stats->min_value->set_notnull();
+            if (table_field->type() == MYSQL_TYPE_BIT)
+              table_field->read_stats->min_value->store(stat_field->val_int(),
+                                                        true);
+            else
+            {
+              stat_field->val_str(&val);
+              table_field->read_stats->min_value->store(val.ptr(),
+                                                        val.length(),
+                                                        &my_charset_bin);
+            }
             break;
           case COLUMN_STAT_MAX_VALUE:
-	    table_field->read_stats->max_value->set_notnull();
-            stat_field->val_str(&val);
-            table_field->read_stats->max_value->store(val.ptr(), val.length(),
-                                                      &my_charset_bin);
+            table_field->read_stats->max_value->set_notnull();
+            if (table_field->type() == MYSQL_TYPE_BIT)
+              table_field->read_stats->max_value->store(stat_field->val_int(),
+                                                        true);
+            else
+            {
+              stat_field->val_str(&val);
+              table_field->read_stats->max_value->store(val.ptr(),
+                                                        val.length(),
+                                                        &my_charset_bin);
+            }
             break;
           case COLUMN_STAT_NULLS_RATIO:
             table_field->read_stats->set_nulls_ratio(stat_field->val_real());