BUG#951283: Wrong result (missing rows) with semijoin+firstmatch, IN/ANY subquery

- The problem was with execution strategy for cases where FirstMatch's inner tables
  were interleaved with outer-uncorrelated tables.
- I was unable to find any cases where such join orders would be practically useful,
  so fixed it by disabling them.

mysql-test/r/subselect_sj2.result

@@ -1043,5 +1043,49 @@ e	e
 e	e
 DROP VIEW v1;
 DROP TABLE t1, t2;
+#
+# BUG#951283: Wrong result (missing rows) with semijoin+firstmatch, IN/ANY subquery
+#
+set @tmp_951283=@@optimizer_prune_level;
+SET optimizer_prune_level=0;
+CREATE TABLE t1 ( a INT ) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(10),(11),(12),(13),(14),(15),(16),
+(17),(18),(19),(20),(21),(22),(23);
+CREATE TABLE t2 (
+b INT PRIMARY KEY,
+c VARCHAR(1),
+d VARCHAR(1),
+KEY(c)
+) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(1,'j','j'),(2,'v','v'),(3,'c','c'),(4,'m','m'),
+(5,'d','d'),(6,'d','d'),(7,'y','y'),(8,'t','t'),
+(9,'d','d'),(10,'s','s'),(11,'r','r'),(12,'m','m'),
+(13,'b','b'),(14,'x','x'),(15,'g','g'),(16,'p','p'),
+(17,'q','q'),(18,'w','w'),(19,'d','d');
+EXPLAIN
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+WHERE alias5.b = alias4.b
+AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	alias3	ALL	PRIMARY	NULL	NULL	NULL	19	Using where
+1	PRIMARY	alias4	ref	PRIMARY,c	c	4	test.alias3.d	1	Using index
+1	PRIMARY	alias5	eq_ref	PRIMARY	PRIMARY	4	test.alias4.b	1	Using where; FirstMatch(alias3)
+1	PRIMARY	alias2	ALL	NULL	NULL	NULL	NULL	14	Using join buffer (flat, BNL join)
+1	PRIMARY	alias1	ALL	NULL	NULL	NULL	NULL	14	Using join buffer (flat, BNL join)
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+WHERE alias5.b = alias4.b
+AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+COUNT(*)
+3724
+set optimizer_prune_level=@tmp_951283;
+DROP TABLE t1,t2;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;

mysql-test/r/subselect_sj2_jcl6.result

@@ -1057,6 +1057,50 @@ x	x
 w	w
 DROP VIEW v1;
 DROP TABLE t1, t2;
+#
+# BUG#951283: Wrong result (missing rows) with semijoin+firstmatch, IN/ANY subquery
+#
+set @tmp_951283=@@optimizer_prune_level;
+SET optimizer_prune_level=0;
+CREATE TABLE t1 ( a INT ) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(10),(11),(12),(13),(14),(15),(16),
+(17),(18),(19),(20),(21),(22),(23);
+CREATE TABLE t2 (
+b INT PRIMARY KEY,
+c VARCHAR(1),
+d VARCHAR(1),
+KEY(c)
+) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(1,'j','j'),(2,'v','v'),(3,'c','c'),(4,'m','m'),
+(5,'d','d'),(6,'d','d'),(7,'y','y'),(8,'t','t'),
+(9,'d','d'),(10,'s','s'),(11,'r','r'),(12,'m','m'),
+(13,'b','b'),(14,'x','x'),(15,'g','g'),(16,'p','p'),
+(17,'q','q'),(18,'w','w'),(19,'d','d');
+EXPLAIN
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+WHERE alias5.b = alias4.b
+AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	alias3	ALL	PRIMARY	NULL	NULL	NULL	19	Using where
+1	PRIMARY	alias4	ref	PRIMARY,c	c	4	test.alias3.d	1	Using index
+1	PRIMARY	alias5	eq_ref	PRIMARY	PRIMARY	4	test.alias4.b	1	Using where; FirstMatch(alias3)
+1	PRIMARY	alias2	ALL	NULL	NULL	NULL	NULL	14	Using join buffer (flat, BNL join)
+1	PRIMARY	alias1	ALL	NULL	NULL	NULL	NULL	14	Using join buffer (incremental, BNL join)
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+WHERE alias5.b = alias4.b
+AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+COUNT(*)
+3724
+set optimizer_prune_level=@tmp_951283;
+DROP TABLE t1,t2;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;
 #
@@ -1078,9 +1122,9 @@ SELECT t3.* FROM t1 JOIN t3 ON t3.b = t1.b
 WHERE c IN (SELECT t4.b FROM t4 JOIN t2);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t3	ALL	NULL	NULL	NULL	NULL	1	Using where
-1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	1	FirstMatch(t3)
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	1	Using join buffer (flat, BNL join)
+1	PRIMARY	t4	ALL	NULL	NULL	NULL	NULL	2	Using where; FirstMatch(t3); Using join buffer (incremental, BNL join)
 1	PRIMARY	t1	ref	b	b	4	test.t3.b	1	Using index
-1	PRIMARY	t4	ALL	NULL	NULL	NULL	NULL	2	Using where; FirstMatch(t1)
 SELECT t3.* FROM t1 JOIN t3 ON t3.b = t1.b
 WHERE c IN (SELECT t4.b FROM t4 JOIN t2);
 b	c
@@ -1106,9 +1150,9 @@ EXPLAIN
 SELECT * FROM t1, t2 WHERE b IN (SELECT a FROM t3, t4 WHERE b = pk);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	1	
-1	PRIMARY	t4	ALL	NULL	NULL	NULL	NULL	1	Using where; FirstMatch(t2)
-1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	
-1	PRIMARY	t3	eq_ref	PRIMARY	PRIMARY	4	test.t4.b	1	Using where; FirstMatch(t1)
+1	PRIMARY	t4	ALL	NULL	NULL	NULL	NULL	1	Using where
+1	PRIMARY	t3	eq_ref	PRIMARY	PRIMARY	4	test.t4.b	1	Using where; FirstMatch(t2)
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using join buffer (flat, BNL join)
 SELECT * FROM t1, t2 WHERE b IN (SELECT a FROM t3, t4 WHERE b = pk);
 pk	a	b
 1	6	8

mysql-test/r/subselect_sj2_mat.result

@@ -1045,6 +1045,50 @@ e	e
 e	e
 DROP VIEW v1;
 DROP TABLE t1, t2;
+#
+# BUG#951283: Wrong result (missing rows) with semijoin+firstmatch, IN/ANY subquery
+#
+set @tmp_951283=@@optimizer_prune_level;
+SET optimizer_prune_level=0;
+CREATE TABLE t1 ( a INT ) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(10),(11),(12),(13),(14),(15),(16),
+(17),(18),(19),(20),(21),(22),(23);
+CREATE TABLE t2 (
+b INT PRIMARY KEY,
+c VARCHAR(1),
+d VARCHAR(1),
+KEY(c)
+) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(1,'j','j'),(2,'v','v'),(3,'c','c'),(4,'m','m'),
+(5,'d','d'),(6,'d','d'),(7,'y','y'),(8,'t','t'),
+(9,'d','d'),(10,'s','s'),(11,'r','r'),(12,'m','m'),
+(13,'b','b'),(14,'x','x'),(15,'g','g'),(16,'p','p'),
+(17,'q','q'),(18,'w','w'),(19,'d','d');
+EXPLAIN
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+WHERE alias5.b = alias4.b
+AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	alias3	ALL	PRIMARY	NULL	NULL	NULL	19	Using where
+1	PRIMARY	alias4	ref	PRIMARY,c	c	4	test.alias3.d	1	Using index
+1	PRIMARY	alias5	eq_ref	PRIMARY	PRIMARY	4	test.alias4.b	1	Using where; FirstMatch(alias3)
+1	PRIMARY	alias2	ALL	NULL	NULL	NULL	NULL	14	Using join buffer (flat, BNL join)
+1	PRIMARY	alias1	ALL	NULL	NULL	NULL	NULL	14	Using join buffer (flat, BNL join)
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+WHERE alias5.b = alias4.b
+AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+COUNT(*)
+3724
+set optimizer_prune_level=@tmp_951283;
+DROP TABLE t1,t2;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;
 set optimizer_switch=default;

mysql-test/t/subselect_sj2.test

@@ -1172,5 +1172,49 @@ SELECT * FROM t2
 
 DROP VIEW v1;
 DROP TABLE t1, t2;
+
+--echo #
+--echo # BUG#951283: Wrong result (missing rows) with semijoin+firstmatch, IN/ANY subquery
+--echo #
+set @tmp_951283=@@optimizer_prune_level;
+SET optimizer_prune_level=0;
+
+CREATE TABLE t1 ( a INT ) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+  (10),(11),(12),(13),(14),(15),(16),
+  (17),(18),(19),(20),(21),(22),(23);
+
+CREATE TABLE t2 (
+  b INT PRIMARY KEY,
+  c VARCHAR(1),
+  d VARCHAR(1),
+  KEY(c)
+) ENGINE=InnoDB;
+
+INSERT INTO t2 VALUES
+  (1,'j','j'),(2,'v','v'),(3,'c','c'),(4,'m','m'),
+  (5,'d','d'),(6,'d','d'),(7,'y','y'),(8,'t','t'),
+  (9,'d','d'),(10,'s','s'),(11,'r','r'),(12,'m','m'),
+  (13,'b','b'),(14,'x','x'),(15,'g','g'),(16,'p','p'),
+  (17,'q','q'),(18,'w','w'),(19,'d','d');
+
+EXPLAIN
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+  SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+  WHERE alias5.b = alias4.b
+    AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+
+SELECT COUNT(*) FROM t1 AS alias1, t1 AS alias2, t2 AS alias3
+WHERE alias3.d IN (
+  SELECT alias4.c FROM t2 AS alias4, t2 AS alias5
+  WHERE alias5.b = alias4.b
+    AND ( alias5.b >= alias3.b OR alias5.c != alias3.c )
+);
+
+set optimizer_prune_level=@tmp_951283;
+DROP TABLE t1,t2;
+
 --echo # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;

sql/opt_subselect.cc

@@ -2902,6 +2902,8 @@ bool Firstmatch_picker::check_qep(JOIN *join,
       }
     }
   }
+  else
+    invalidate_firstmatch_prefix();
   return FALSE;
 }