MDEV-10466 Server crashed in SEL_ARG::store_min() with extended_keys=on

This bug could manifest itself in a very rare cases when the optimizer
chose an execution plan by which a joined table was accessed by a table
scan and the optimizer was checking whether ranges checked for each record
could improve this plan. In such cases the optimizer evaluates range
conditions over a table that depend on other tables. For such conditions
the constructed SEL_ARG trees are marked as MAYBE_KEY. If a SEL_ARG object
constructed for a sargable condition marked as RANGE_KEY had the same
first key part as a MAYBE_KEY SEL_ARG object and the key_and() function
was called for this pair of SEL_ARG objects then an invalid SEL_ARG
object could be constructed that ultimately could lead to a crash before
the execution phase.

mysql-test/r/range_innodb.result

@@ -37,3 +37,28 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t0	ALL	NULL	NULL	NULL	NULL	10	
 1	SIMPLE	t2	range	a,b	b	5	NULL	201	Using where; Using join buffer (flat, BNL join)
 drop table t0,t1,t2;
+#
+# MDEV-10466: constructing an invalid SEL_ARG
+#
+create table t1 (
+pk int, a int, b int,
+primary key (pk), index idx1(b), index idx2(b)
+) engine=innodb;
+insert into t1 values (1,6,0),(2,1,0),(3,5,2),(4,8,0);
+create table t2 (c int) engine=innodb;
+insert into t2 values (1),(2);
+create table t3 (d int) engine=innodb;
+insert into t3 values (3),(-1),(4);
+set @save_optimizer_switch=@@optimizer_switch;
+set optimizer_switch='extended_keys=on';
+explain
+select pk, a, b from t1,t2,t3 where b >= d and pk < c and b = '0';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (flat, BNL join)
+1	SIMPLE	t1	ALL	PRIMARY,idx1,idx2	NULL	NULL	NULL	4	Using where; Using join buffer (incremental, BNL join)
+select pk, a, b from t1,t2,t3 where b >= d and pk < c and b = '0';
+pk	a	b
+1	6	0
+set optimizer_switch=@save_optimizer_switch;
+drop table t1,t2,t3;

mysql-test/t/range_innodb.test

@@ -45,3 +45,28 @@ explain select * from t0 left join t2 on t2.a <t0.a and t2.b between 50 and 250;
 
 drop table t0,t1,t2;
 
+
+--echo #
+--echo # MDEV-10466: constructing an invalid SEL_ARG
+--echo #
+
+create table t1 (
+  pk int, a int, b int,
+  primary key (pk), index idx1(b), index idx2(b)
+) engine=innodb;
+insert into t1 values (1,6,0),(2,1,0),(3,5,2),(4,8,0);
+create table t2 (c int) engine=innodb;
+insert into t2 values (1),(2);
+create table t3 (d int) engine=innodb;
+insert into t3 values (3),(-1),(4);
+
+set @save_optimizer_switch=@@optimizer_switch;
+set optimizer_switch='extended_keys=on';
+
+explain
+select pk, a, b from t1,t2,t3 where b >= d and pk < c and b = '0';
+select pk, a, b from t1,t2,t3 where b >= d and pk < c and b = '0';
+
+set optimizer_switch=@save_optimizer_switch;
+
+drop table t1,t2,t3;

sql/opt_range.cc

@@ -8837,7 +8837,7 @@ key_and(RANGE_OPT_PARAM *param, SEL_ARG *key1, SEL_ARG *key2, uint clone_flag)
       if (key2->next_key_part)
       {
 	key1->use_count--;			// Incremented in and_all_keys
-	return and_all_keys(param, key1, key2, clone_flag);
+        return and_all_keys(param, key1, key2->next_key_part, clone_flag);
       }
       key2->use_count--;			// Key2 doesn't have a tree
     }