BUG#861147: Assertion `fixed == 1' failed in Item_func_eq::val_int() with...

BUG#861147: Assertion `fixed == 1' failed in Item_func_eq::val_int() with semijoin + materialization
- convert_subq_to_jtbm() didn't check that subuqery optimization was successful. If it wasn't (in this
  example because of @@max_join_size violation), it would proceed further and eventually crash when 
  trying to execute the un-optimized subquery.

mysql-test/r/subselect_sj.result

@@ -1920,4 +1920,46 @@ WHERE t2.b NOT IN (SELECT t4.b FROM t4 WHERE t4.b < t1.b)
 a
 8
 DROP TABLE t1, t2, t4, t5;
+#
+# BUG#861147: Assertion `fixed == 1' failed in Item_func_eq::val_int() with semijoin + materialization + max_join_size
+#
+CREATE TABLE t1 ( f2 int) ;
+CREATE TABLE t2 ( f1 int, f3 int, f4 varchar(3), f5 varchar(35)) ;
+INSERT INTO t2 VALUES (4057,9,'USA','Visalia'),(3993,11,'USA','Waco'),
+(3948,14,'USA','Warren'),(3813,57,'USA','Washington'),
+(4010,11,'USA','Waterbury'),(4017,11,'USA','West Covina'),
+(4004,11,'USA','West Valley City'),(4033,10,'USA','Westminster'),
+(3842,34,'USA','Wichita'),(4018,10,'USA','Wichita Falls'),
+(3899,19,'USA','Winston-Salem'),(3914,17,'USA','Worcester'),
+(3888,20,'USA','Yonkers');
+CREATE TABLE t3 ( f3 int, f4 varchar(3)) ;
+INSERT INTO t3 VALUES (86,'USA');
+CREATE TABLE t4 ( f3 int, f4 varchar(3), f5 varchar(52)) ;
+INSERT INTO t4 VALUES (0,'RUS','Belorussian'),(0,'USA','Portuguese');
+CREATE TABLE t5 ( f2 int) ;
+CREATE TABLE t6 ( f4 varchar(3));
+INSERT INTO t6 VALUES ('RUS'),('USA');
+set @tmp_mjs_861147= @@max_join_size;
+SET max_join_size=10;
+set @tmp_os_861147= @@optimizer_switch;
+set @@optimizer_switch='semijoin=on,materialization=on';
+SELECT *
+FROM t1
+WHERE ( 1 , 3 ) IN (
+SELECT t2.f1 , MAX( t3.f3 )
+FROM t2
+JOIN t3
+WHERE t3.f4 IN (
+SELECT t4.f5
+FROM t4
+STRAIGHT_JOIN t5
+WHERE t4.f4 < t2.f5
+)
+) AND ( 'p' , 'k' ) IN (
+SELECT f4 , f4 FROM t6
+);
+ERROR 42000: The SELECT would examine more than MAX_JOIN_SIZE rows; check your WHERE and use SET SQL_BIG_SELECTS=1 or SET SQL_MAX_JOIN_SIZE=# if the SELECT is okay
+set max_join_size= @tmp_mjs_861147;
+set optimizer_switch= @tmp_os_861147;
+DROP TABLE t1,t2,t3,t4,t5,t6;
 set optimizer_switch=@subselect_sj_tmp;

mysql-test/r/subselect_sj_jcl6.result

@@ -1931,6 +1931,48 @@ WHERE t2.b NOT IN (SELECT t4.b FROM t4 WHERE t4.b < t1.b)
 a
 8
 DROP TABLE t1, t2, t4, t5;
+#
+# BUG#861147: Assertion `fixed == 1' failed in Item_func_eq::val_int() with semijoin + materialization + max_join_size
+#
+CREATE TABLE t1 ( f2 int) ;
+CREATE TABLE t2 ( f1 int, f3 int, f4 varchar(3), f5 varchar(35)) ;
+INSERT INTO t2 VALUES (4057,9,'USA','Visalia'),(3993,11,'USA','Waco'),
+(3948,14,'USA','Warren'),(3813,57,'USA','Washington'),
+(4010,11,'USA','Waterbury'),(4017,11,'USA','West Covina'),
+(4004,11,'USA','West Valley City'),(4033,10,'USA','Westminster'),
+(3842,34,'USA','Wichita'),(4018,10,'USA','Wichita Falls'),
+(3899,19,'USA','Winston-Salem'),(3914,17,'USA','Worcester'),
+(3888,20,'USA','Yonkers');
+CREATE TABLE t3 ( f3 int, f4 varchar(3)) ;
+INSERT INTO t3 VALUES (86,'USA');
+CREATE TABLE t4 ( f3 int, f4 varchar(3), f5 varchar(52)) ;
+INSERT INTO t4 VALUES (0,'RUS','Belorussian'),(0,'USA','Portuguese');
+CREATE TABLE t5 ( f2 int) ;
+CREATE TABLE t6 ( f4 varchar(3));
+INSERT INTO t6 VALUES ('RUS'),('USA');
+set @tmp_mjs_861147= @@max_join_size;
+SET max_join_size=10;
+set @tmp_os_861147= @@optimizer_switch;
+set @@optimizer_switch='semijoin=on,materialization=on';
+SELECT *
+FROM t1
+WHERE ( 1 , 3 ) IN (
+SELECT t2.f1 , MAX( t3.f3 )
+FROM t2
+JOIN t3
+WHERE t3.f4 IN (
+SELECT t4.f5
+FROM t4
+STRAIGHT_JOIN t5
+WHERE t4.f4 < t2.f5
+)
+) AND ( 'p' , 'k' ) IN (
+SELECT f4 , f4 FROM t6
+);
+ERROR 42000: The SELECT would examine more than MAX_JOIN_SIZE rows; check your WHERE and use SET SQL_BIG_SELECTS=1 or SET SQL_MAX_JOIN_SIZE=# if the SELECT is okay
+set max_join_size= @tmp_mjs_861147;
+set optimizer_switch= @tmp_os_861147;
+DROP TABLE t1,t2,t3,t4,t5,t6;
 set optimizer_switch=@subselect_sj_tmp;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off

mysql-test/t/subselect_sj.test

@@ -1756,5 +1756,56 @@ select * from t5 where (a) in (
 
 DROP TABLE t1, t2, t4, t5;
 
+--echo #
+--echo # BUG#861147: Assertion `fixed == 1' failed in Item_func_eq::val_int() with semijoin + materialization + max_join_size
+--echo #
+CREATE TABLE t1 ( f2 int) ;
+CREATE TABLE t2 ( f1 int, f3 int, f4 varchar(3), f5 varchar(35)) ;
+INSERT INTO t2 VALUES (4057,9,'USA','Visalia'),(3993,11,'USA','Waco'),
+  (3948,14,'USA','Warren'),(3813,57,'USA','Washington'),
+  (4010,11,'USA','Waterbury'),(4017,11,'USA','West Covina'),
+  (4004,11,'USA','West Valley City'),(4033,10,'USA','Westminster'),
+  (3842,34,'USA','Wichita'),(4018,10,'USA','Wichita Falls'),
+  (3899,19,'USA','Winston-Salem'),(3914,17,'USA','Worcester'),
+  (3888,20,'USA','Yonkers');
+
+CREATE TABLE t3 ( f3 int, f4 varchar(3)) ;
+INSERT INTO t3 VALUES (86,'USA');
+
+CREATE TABLE t4 ( f3 int, f4 varchar(3), f5 varchar(52)) ;
+INSERT INTO t4 VALUES (0,'RUS','Belorussian'),(0,'USA','Portuguese');
+
+CREATE TABLE t5 ( f2 int) ;
+
+CREATE TABLE t6 ( f4 varchar(3));
+INSERT INTO t6 VALUES ('RUS'),('USA');
+
+set @tmp_mjs_861147= @@max_join_size;
+SET max_join_size=10;
+set @tmp_os_861147= @@optimizer_switch;
+set @@optimizer_switch='semijoin=on,materialization=on';
+
+--error ER_TOO_BIG_SELECT
+SELECT *
+FROM t1
+WHERE ( 1 , 3 ) IN (
+        SELECT t2.f1 , MAX( t3.f3 )
+        FROM t2
+        JOIN t3
+        WHERE t3.f4 IN (
+                SELECT t4.f5
+                FROM t4
+                STRAIGHT_JOIN t5
+                WHERE t4.f4 < t2.f5
+        )
+) AND ( 'p' , 'k' ) IN (
+        SELECT f4 , f4 FROM t6
+);
+set max_join_size= @tmp_mjs_861147;
+set optimizer_switch= @tmp_os_861147;
+
+DROP TABLE t1,t2,t3,t4,t5,t6;
+
+
 # The following command must be the last one the file 
 set optimizer_switch=@subselect_sj_tmp;

sql/opt_subselect.cc

@@ -1425,7 +1425,8 @@ static bool convert_subq_to_jtbm(JOIN *parent_join,
   DBUG_ENTER("convert_subq_to_jtbm");
 
   subq_pred->in_strategy &= ~SUBS_IN_TO_EXISTS;
-  subq_pred->optimize(&rows, &read_time);
+  if (subq_pred->optimize(&rows, &read_time))
+    DBUG_RETURN(TRUE);
 
   subq_pred->jtbm_read_time= read_time;
   subq_pred->jtbm_record_count=rows;