Fixed bug #16203.

If check_quick_select returns non-empty range then the function cost_group_min_max
cannot return 0 as an estimate of the number of retrieved records.
Yet the function erroneously returned 0 as the estimate in some situations.

mysql-test/r/group_min_max.result

@@ -2043,3 +2043,30 @@ c1	c2
 30	8
 30	9
 drop table t1;
+CREATE TABLE t1 (a varchar(5), b int(11), PRIMARY KEY (a,b));
+INSERT INTO t1 VALUES ('AA',1), ('AA',2), ('AA',3), ('BB',1), ('AA',4);
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+SELECT a FROM t1 WHERE a='AA' GROUP BY a;
+a
+AA
+SELECT a FROM t1 WHERE a='BB' GROUP BY a;
+a
+BB
+EXPLAIN SELECT a FROM t1 WHERE a='AA' GROUP BY a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	PRIMARY	PRIMARY	7	const	3	Using where; Using index
+EXPLAIN SELECT a FROM t1 WHERE a='BB' GROUP BY a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	PRIMARY	PRIMARY	7	const	1	Using where; Using index
+SELECT DISTINCT a FROM t1 WHERE a='BB';
+a
+BB
+SELECT DISTINCT a FROM t1 WHERE a LIKE 'B%';
+a
+BB
+SELECT a FROM t1 WHERE a LIKE 'B%' GROUP BY a;
+a
+BB
+DROP TABLE t1;

mysql-test/t/group_min_max.test

@@ -715,3 +715,24 @@ select distinct c1, c2 from t1 order by c2;
 select c1,min(c2) as c2 from t1 group by c1 order by c2;
 select c1,c2 from t1 group by c1,c2 order by c2;
 drop table t1;
+
+#
+# Bug #16203: Analysis for possible min/max optimization erroneously
+#             returns impossible range
+#
+
+CREATE TABLE t1 (a varchar(5), b int(11), PRIMARY KEY (a,b));
+INSERT INTO t1 VALUES ('AA',1), ('AA',2), ('AA',3), ('BB',1), ('AA',4);
+OPTIMIZE TABLE t1;
+
+SELECT a FROM t1 WHERE a='AA' GROUP BY a;
+SELECT a FROM t1 WHERE a='BB' GROUP BY a;
+
+EXPLAIN SELECT a FROM t1 WHERE a='AA' GROUP BY a;
+EXPLAIN SELECT a FROM t1 WHERE a='BB' GROUP BY a;
+
+SELECT DISTINCT a FROM t1 WHERE a='BB';
+SELECT DISTINCT a FROM t1 WHERE a LIKE 'B%';
+SELECT a FROM t1 WHERE a LIKE 'B%' GROUP BY a;
+
+DROP TABLE t1;

sql/opt_range.cc

@@ -7780,6 +7780,7 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts,
     quick_prefix_selectivity= (double) quick_prefix_records /
                               (double) table_records;
     num_groups= (uint) rint(num_groups * quick_prefix_selectivity);
+    set_if_bigger(num_groups, 1);
   }
 
   if (used_key_parts > group_key_parts)