Fix LP BUG#772309

Analysis:
The method st_select_lex::optimize_unflattened_subqueries()
incorrectly propagated to each subquery the complete
select_options flag set for the whole query. Among other
flags in select_options, this propagated incorrectly the
STRAIGHT_JOIN flag from the upper query to the subquery.

Solution:
During EXPLAIN set only the SELECT_DESCRIBE bit in the
select_options of the subquery.

mysql-test/r/subselect4.result

@@ -15,7 +15,7 @@ ORDER BY count(*);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	index	NULL	a	5	NULL	2	Using where; Using index; Using temporary
 2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
-3	DEPENDENT SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	no matching row in const table
+3	DEPENDENT SUBQUERY	t3	system	NULL	NULL	NULL	NULL	0	const row not found
 # should not crash the next statement
 SELECT 1 FROM t1
 WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE 1 = (SELECT MIN(t2.b) FROM t3))
@@ -1718,3 +1718,28 @@ SELECT * FROM t1 WHERE a1 IN (SELECT b1 FROM t2 WHERE b1 = b2);
 a1	a2
 set @@optimizer_switch=@save_optimizer_switch;
 drop table t1, t2;
+#                                                                                                                                                     
+# LP BUG#772309 join_tab_cmp_straight(): Assertion `!jt2->emb_sj_nest' failed in maria-5.3-mwl89 with semijoin
+#                                                                                                                                                     
+CREATE TABLE t1 ( f2 int) ;
+INSERT INTO t1 VALUES (0),(0);
+CREATE TABLE t2 ( f1 int NOT NULL ) ;
+INSERT INTO t2 VALUES (0),(0);
+CREATE TABLE t3 ( f1 int NOT NULL , f2 int) ;
+INSERT INTO t3 VALUES (0,0), (0,0);
+EXPLAIN SELECT STRAIGHT_JOIN (
+SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	DEPENDENT SUBQUERY	t3	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+SELECT STRAIGHT_JOIN (
+SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+(
+SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+)
+NULL
+drop table t1, t2, t3;

mysql-test/r/union.result

@@ -1632,7 +1632,7 @@ ORDER BY (SELECT a FROM t2 WHERE b = 12);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	UNION	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	
-NULL	UNION	<union1,2>	ALL	NULL	NULL	NULL	NULL	NULL	NULL	Using filesort
+NULL	UNION RESULT	<union1,2>	ALL	NULL	NULL	NULL	NULL	NULL	NULL	Using filesort
 3	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 Warnings:
 Note	1276	Field or reference 'a' of SELECT #3 was resolved in SELECT #-1

mysql-test/t/subselect4.test

@@ -1390,3 +1390,25 @@ SELECT * FROM t1 WHERE a1 IN (SELECT b1 FROM t2 WHERE b1 = b2);
 
 set @@optimizer_switch=@save_optimizer_switch;
 drop table t1, t2;
+
+--echo #                                                                                                                                                     
+--echo # LP BUG#772309 join_tab_cmp_straight(): Assertion `!jt2->emb_sj_nest' failed in maria-5.3-mwl89 with semijoin
+--echo #                                                                                                                                                     
+
+CREATE TABLE t1 ( f2 int) ;
+INSERT INTO t1 VALUES (0),(0);
+
+CREATE TABLE t2 ( f1 int NOT NULL ) ;
+INSERT INTO t2 VALUES (0),(0);
+
+CREATE TABLE t3 ( f1 int NOT NULL , f2 int) ;
+INSERT INTO t3 VALUES (0,0), (0,0);
+
+EXPLAIN SELECT STRAIGHT_JOIN (
+        SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+SELECT STRAIGHT_JOIN (
+        SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+
+drop table t1, t2, t3;

sql/sql_lex.cc

@@ -3136,8 +3136,9 @@ bool st_select_lex::optimize_unflattened_subqueries()
         if (un->outer_select()->options & SELECT_DESCRIBE)
         {
           /* Optimize the subquery in the context of EXPLAIN. */
-          set_explain_type();
-          inner_join->select_options= options;
+          sl->set_explain_type();
+          sl->options|= SELECT_DESCRIBE;
+          inner_join->select_options|= SELECT_DESCRIBE;
         }
         res= inner_join->optimize();
         inner_join->select_options= save_options;

sql/sql_select.cc

@@ -19898,7 +19898,10 @@ bool mysql_explain_union(THD *thd, SELECT_LEX_UNIT *unit, select_result *result)
   SELECT_LEX *first= unit->first_select();
 
   for (SELECT_LEX *sl= first; sl; sl= sl->next_select())
+  {
     sl->set_explain_type();
+    sl->options|= SELECT_DESCRIBE;
+  }
 
   if (unit->is_union())
   {