MDEV-30569: Assertion ...ha_table_flags() in Duplicate_weedout_picker::check_qep

DuplicateWeedout semi-join optimization requires that the tables in
the parent subquery provide rowids that can be compared across table
scans. Most engines support this, federated is the only exception.

DuplicateWeedout is the default catch-all semi-join strategy, which
must be always available. If it is not available for some edge case,
it's better to disable semi-join conversion altogether.

This is what was done in the fix for MDEV-30395. However that fix
has put the check before the view processing, so it didn't detect
federated tables inside mergeable VIEWs.

This patch moves the check to be done at a later phase, when mergeable
views are already merged.

mysql-test/suite/federated/federatedx.result

@@ -2357,6 +2357,21 @@ DROP TABLE t2_fed, t1, t2;
 set @@optimizer_switch=@save_optimizer_switch;
 DROP SERVER s;
 # End of 10.5 tests
+#
+# MDEV-30569: Assertion ...ha_table_flags() failed in Duplicate_weedout_picker::check_qep
+#
+create server s foreign data wrapper mysql options
+(host "127.0.0.1", database "test", user "root", port $MASTER_MYPORT);
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+CREATE TABLE t1_fed ENGINE=FEDERATED CONNECTION='s/t1';
+CREATE VIEW v AS SELECT * FROM t1_fed;
+SELECT * FROM v WHERE a IN ( SELECT b FROM t2);
+a
+DROP TABLE t1_fed, t1, t2;
+DROP SERVER s;
 connection master;
 DROP TABLE IF EXISTS federated.t1;
 DROP DATABASE IF EXISTS federated;

mysql-test/suite/federated/federatedx.test

@@ -2090,4 +2090,25 @@ DROP SERVER s;
 
 --echo # End of 10.5 tests
 
+--echo #
+--echo # MDEV-30569: Assertion ...ha_table_flags() failed in Duplicate_weedout_picker::check_qep
+--echo #
+
+evalp create server s foreign data wrapper mysql options
+  (host "127.0.0.1", database "test", user "root", port $MASTER_MYPORT);
+
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+
+CREATE TABLE t1_fed ENGINE=FEDERATED CONNECTION='s/t1';
+CREATE VIEW v AS SELECT * FROM t1_fed;
+
+SELECT * FROM v WHERE a IN ( SELECT b FROM t2);
+
+DROP TABLE t1_fed, t1, t2;
+DROP SERVER s;
+
 source include/federated_cleanup.inc;

sql/opt_subselect.cc

@@ -666,17 +666,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         DBUG_RETURN(-1);
       }
     }
-    /* Check if any table is not supporting comparable rowids */
-    {
-      List_iterator_fast<TABLE_LIST> li(select_lex->outer_select()->leaf_tables);
-      TABLE_LIST *tbl;
-      while ((tbl = li++))
-      {
-        TABLE *table= tbl->table;
-        if (table && table->file->ha_table_flags() & HA_NON_COMPARABLE_ROWID)
-          join->not_usable_rowid_map|= table->map;
-      }
-    }
 
     DBUG_PRINT("info", ("Checking if subq can be converted to semi-join"));
     /*
@@ -698,9 +687,10 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         11. It is first optimisation (the subquery could be moved from ON
             clause during first optimisation and then be considered for SJ
             on the second when it is too late)
-        12. All tables supports comparable rowids.
-            This is needed for DuplicateWeedout strategy to work (which
-            is the catch-all semi-join strategy so it must be applicable).
+
+      There are also other requirements which cannot be checked at this phase,
+      yet. They are checked later in convert_join_subqueries_to_semijoins(),
+      look for calls to block_conversion_to_sj().
     */
     if (optimizer_flag(thd, OPTIMIZER_SWITCH_SEMIJOIN) &&
         in_subs &&                                                    // 1
@@ -715,8 +705,7 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         !((join->select_options |                                     // 10
            select_lex->outer_select()->join->select_options)          // 10
           & SELECT_STRAIGHT_JOIN) &&                                  // 10
-        select_lex->first_cond_optimization &&                        // 11
-        join->not_usable_rowid_map == 0)                              // 12
+        select_lex->first_cond_optimization)                          // 11
     {
       DBUG_PRINT("info", ("Subquery is semi-join conversion candidate"));
 
@@ -1230,7 +1219,36 @@ bool convert_join_subqueries_to_semijoins(JOIN *join)
     }
   }
 
-  if (join->select_options & SELECT_STRAIGHT_JOIN)
+  /*
+    Compute join->not_usable_rowid_map.
+    The idea is:
+    - DuplicateWeedout strategy requires that one is able to get the rowid
+      (call h->position()) for tables in the parent select. Obtained Rowid
+      values must be stable across table scans.
+      = Rowids are typically available. The only known exception is federatedx
+        tables.
+    - The optimizer requires that DuplicateWeedout strategy is always
+      applicable. It is the only strategy that is applicable for any join
+      order. The optimizer is not prepared for the situation where it has
+      constructed a join order and then it turns out that there's no semi-join
+      strategy that can be used for it.
+
+    Because of the above, we will not use semi-joins if the parent select has
+    tables which do not support rowids.
+  */
+  {
+    List_iterator_fast<TABLE_LIST> li(join->select_lex->leaf_tables);
+    TABLE_LIST *tbl;
+    while ((tbl = li++))
+    {
+      TABLE *table= tbl->table;
+      if (table && table->file->ha_table_flags() & HA_NON_COMPARABLE_ROWID)
+        join->not_usable_rowid_map|= table->map;
+    }
+  }
+
+  if (join->select_options & SELECT_STRAIGHT_JOIN ||
+      join->not_usable_rowid_map != 0)
   {
     /* Block conversion to semijoins for all candidates */ 
     li.rewind();