MDEV-26247 Re-implement spider gbh query rewrite of tables

Spider GBH's query rewrite of table joins is overly complex and
error-prone. We replace it with something closer to what
dbug_print() (more specifically, print_join()) does, but catered to
spider.

More specifically, we replace the body of
spider_db_mbase_util::append_from_and_tables() with a call to
spider_db_mbase_util::append_join(), and remove downstream append_X
functions.

We make it handle const tables by rewriting them as (select 1). This
fixes the main issue in MDEV-26247.

We also ban semijoin from spider gbh, which fixes MDEV-31645 and
MDEV-30392, as semi-join is an "internal" join, and "semi join" does
not parse, and it is different from "join" in that it deduplicates the
right hand side

Not all queries passed to a group by handler are valid (MDEV-32273),
for example, a join on expr may refer outer fields not in the current
context. We detect this during the handler creation when walking the
join. See also gbh_outer_fields_in_join.test.

It also skips eliminated tables, which fixes MDEV-26193.

storage/spider/mysql-test/spider/bugfix/r/gbh_outer_fields_in_join.result

+#
+# MDEV-26247 Spider: Valid LEFT JOIN results in ERROR 1064
+#
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+connection child2_1;
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+CREATE TABLE t1 (a int) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+CREATE TABLE t2 (a int) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+CREATE TABLE t3 (a int) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1), (2);
+INSERT INTO t3 VALUES (1), (2), (3);
+connection master_1;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+CREATE TABLE t1 (a int) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t1"';
+CREATE TABLE t2 (a int) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t2"';
+CREATE TABLE t3 (a int) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t3"';
+select * from t3 left join t1 on t3.a = t1.a left join t2 on t3.a = t2.a;
+a	a	a
+1	1	1
+2	NULL	2
+3	NULL	NULL
+select * from t1 left join t2 on t1.a = t2.a right join t3 on t3.a = t1.a;
+a	a	a
+1	1	1
+NULL	NULL	2
+NULL	NULL	3
+select * from t3 left join (t1 left join t2 on t1.a = t2.a) on t3.a = t1.a;
+a	a	a
+1	1	1
+2	NULL	NULL
+3	NULL	NULL
+drop table t1, t2, t3;
+connection master_1;
+DROP DATABASE IF EXISTS auto_test_local;
+connection child2_1;
+DROP DATABASE IF EXISTS auto_test_remote;
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3

storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result

@@ -72,6 +72,7 @@ SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
 select `pkey`,`val` from `auto_test_remote`.`tbl_a`
 select `pkey`,`val` from `auto_test_remote`.`tbl_a` where `pkey` = 1
+select 1 from (select 1) t0
 select `pkey`,`val` from `auto_test_remote`.`tbl_a`
 select `pkey`,`val` from `auto_test_remote`.`tbl_a`
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
@@ -85,6 +86,7 @@ argument
 select `pkey`,`val` from `auto_test_remote2`.`tbl_a`
 select `pkey`,`val` from `auto_test_remote2`.`tbl_a`
 select `pkey`,`val` from `auto_test_remote2`.`tbl_a` where `pkey` = 2
+select 1 from (select 1) t0
 select `pkey`,`val` from `auto_test_remote2`.`tbl_a`
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT pkey, val FROM tbl_a ORDER BY pkey;

storage/spider/mysql-test/spider/bugfix/r/mdev_26247.result

+#
+# MDEV-26247 Spider: Valid LEFT JOIN results in ERROR 1064
+#
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+connection child2_1;
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+CREATE TABLE t1 (
+a11 int,
+primary key (a11)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+CREATE TABLE t2 (
+a21 int,
+a22 int,
+primary key (a21, a22)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+CREATE TABLE t3 (
+a31 int,
+a32 int,
+primary key (a31, a32)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+INSERT INTO t1 VALUES (1), (2), (3);
+INSERT INTO t2 VALUES (1, 11), (2, 22), (3, 33);
+INSERT INTO t3 VALUES (1, 111), (2, 222), (3, 333);
+connection master_1;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+CREATE TABLE t1 (
+a11 int,
+primary key (a11)
+) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t1"';
+CREATE TABLE t2 (
+a21 int,
+a22 int,
+primary key (a21, a22)
+) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t2"';
+CREATE TABLE t3 (
+a31 int,
+a32 int,
+primary key (a31, a32)
+) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t3"';
+SELECT a22 FROM t1 LEFT JOIN t2 ON a11 = a21 WHERE a11 IN (1);
+a22
+11
+SELECT a22 FROM t1 LEFT JOIN t2 ON a11 = a21 - a21 WHERE a11 IN (1);
+a22
+NULL
+SELECT a22 FROM t2 RIGHT JOIN t1 ON a21 = a11 WHERE a11 IN (1);
+a22
+11
+SELECT a22 FROM t2 RIGHT JOIN t1 ON a21 = a11 WHERE a11 IN (1,2);
+a22
+11
+22
+SELECT a22, a32 FROM t1 LEFT JOIN t2 ON a11 = a21 RIGHT JOIN t3 on a21 = a31 WHERE a11 IN (1);
+a22	a32
+11	111
+SELECT a22, a32 FROM t1 LEFT JOIN t2 ON a11 = a21 - a21 RIGHT JOIN t3 on a21 = a31 - a31 WHERE a11 IN (1);
+a22	a32
+SELECT a22, a32 FROM t1 LEFT JOIN t2 ON a11 = a21 RIGHT JOIN t3 on a21 = a31 WHERE a11 IN (1,2);
+a22	a32
+11	111
+22	222
+SELECT a22 FROM t2 LEFT JOIN t1 ON a11 = a21 WHERE a11 IN (1);
+a22
+11
+SELECT a22 FROM t2 LEFT JOIN t1 ON a11 = a21 - a21 WHERE a11 IN (1);
+a22
+SELECT a22 FROM t1 RIGHT JOIN t2 ON a21 = a11 WHERE a11 IN (1);
+a22
+11
+SELECT a22 FROM t1 RIGHT JOIN t2 ON a21 = a11 WHERE a11 IN (1,2);
+a22
+11
+22
+connection master_1;
+DROP DATABASE IF EXISTS auto_test_local;
+connection child2_1;
+DROP DATABASE IF EXISTS auto_test_remote;
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3

storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result

+#
+# MDEV-29163 Server crash with SIGSEGV or dynamic-stack-buffer-overflow in spider_db_mbase_util::append_table
+#
+for master_1
+for child2
+for child3
+CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (b INT);
+CREATE TABLE t3 (c INT, PRIMARY KEY(c));
+CREATE TABLE t1_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't1'";
+CREATE TABLE t2_spider (b INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't2'";
+CREATE TABLE t3_spider (c INT, PRIMARY KEY(c)) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't3'";
+SELECT t1_spider.* FROM t1_spider LEFT JOIN t2_spider LEFT JOIN t3_spider ON b = c ON a = b;
+a
+DROP TABLE t1_spider, t2_spider, t3_spider, t1, t2, t3;
+drop server s;
+for master_1
+for child2
+for child3
+#
+# end of test mdev_29163
+#

storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result

+#
+# MDEV-30392 Syntax error upon query with subquery from Spider table
+#
+for master_1
+for child2
+for child3
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 'srv', table 't1'";
+SELECT a FROM t2 WHERE a IN ( SELECT a FROM t2 );
+a
+1
+2
+DROP TABLE t1, t2;
+DROP SERVER srv;
+for master_1
+for child2
+for child3
+#
+# end of test mdev_30392
+#

storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result

+#
+# MDEV-31645 Spider doesn't recognize semi JOIN
+#
+for master_1
+for child2
+for child3
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+CREATE TABLE t1 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a));
+CREATE TABLE t2 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)) ENGINE=SPIDER COMMENT='srv "srv", WRAPPER "mysql", TABLE "t1"';
+SELECT * FROM t2 WHERE b IN (SELECT b FROM t2 WHERE a > 10);
+a	b
+SELECT * FROM t2
+WHERE A BETWEEN 0 AND 10 AND B IN(SELECT B FROM t2 WHERE A BETWEEN 11 AND 20);
+a	b
+drop table t1, t2;
+drop server srv;
+for master_1
+for child2
+for child3
+#
+# end of test mdev_31645
+#

storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result

+#
+# Test joining a spider table with a non-spider table
+#
+for master_1
+for child2
+for child3
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+create table t1 (c int);
+create table t2 (d int);
+insert into t2 values (1), (2);
+create table t3 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"';
+insert into t3 values (2), (3);
+select c from t3 join t2 on c = d;
+c
+2
+drop table t1, t2, t3;
+drop server srv;
+for master_1
+for child2
+for child3
+#
+# end of test spider_join_with_non_spider
+#

storage/spider/mysql-test/spider/bugfix/r/subquery.result

+#
+# Test spider select with subqueries
+#
+for master_1
+for child2
+for child3
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+create table t1 (c1 int);
+create table t2 (c2 int);
+insert into t1 values (1), (2);
+insert into t2 values (0), (1), (2);
+create table t1s (c1 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"';
+create table t2s (c2 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
+select c1 from t1s, (select c2 from t2s where c2 > 0) t where c1 + 1 = c2;
+c1
+1
+drop table t1, t2, t1s, t2s;
+drop server srv;
+for master_1
+for child2
+for child3
+#
+# end of test subquery
+#

storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.cnf

+!include include/default_mysqld.cnf
+!include ../my_1_1.cnf
+!include ../my_2_1.cnf

storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.test

+--echo #
+--echo # MDEV-26247 Spider: Valid LEFT JOIN results in ERROR 1064
+--echo #
+
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+
+--connection child2_1
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+
+eval CREATE TABLE t1 (a int) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+eval CREATE TABLE t2 (a int) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+eval CREATE TABLE t3 (a int) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1), (2);
+INSERT INTO t3 VALUES (1), (2), (3);
+
+--connection master_1
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+
+eval CREATE TABLE t1 (a int) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t1"';
+eval CREATE TABLE t2 (a int) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t2"';
+eval CREATE TABLE t3 (a int) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t3"';
+
+select * from t3 left join t1 on t3.a = t1.a left join t2 on t3.a = t2.a;
+select * from t1 left join t2 on t1.a = t2.a right join t3 on t3.a = t1.a;
+select * from t3 left join (t1 left join t2 on t1.a = t2.a) on t3.a = t1.a;
+
+drop table t1, t2, t3;
+
+--connection master_1
+DROP DATABASE IF EXISTS auto_test_local;
+--connection child2_1
+DROP DATABASE IF EXISTS auto_test_remote;
+
+--disable_query_log
+--disable_result_log
+--source ../t/test_deinit.inc
+--enable_query_log
+--enable_result_log

storage/spider/mysql-test/spider/bugfix/t/mdev_26247.cnf

+!include include/default_mysqld.cnf
+!include ../my_1_1.cnf
+!include ../my_2_1.cnf

storage/spider/mysql-test/spider/bugfix/t/mdev_26247.test

+--echo #
+--echo # MDEV-26247 Spider: Valid LEFT JOIN results in ERROR 1064
+--echo #
+
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+
+--connection child2_1
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+
+eval CREATE TABLE t1 (
+  a11 int,
+  primary key (a11)
+) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+
+eval CREATE TABLE t2 (
+  a21 int,
+  a22 int,
+  primary key (a21, a22)
+) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+
+eval CREATE TABLE t3 (
+  a31 int,
+  a32 int,
+  primary key (a31, a32)
+) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+
+INSERT INTO t1 VALUES (1), (2), (3);
+INSERT INTO t2 VALUES (1, 11), (2, 22), (3, 33);
+INSERT INTO t3 VALUES (1, 111), (2, 222), (3, 333);
+
+--connection master_1
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+
+eval CREATE TABLE t1 (
+  a11 int,
+  primary key (a11)
+) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t1"';
+
+eval CREATE TABLE t2 (
+  a21 int,
+  a22 int,
+  primary key (a21, a22)
+) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t2"';
+
+eval CREATE TABLE t3 (
+  a31 int,
+  a32 int,
+  primary key (a31, a32)
+) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t3"';
+
+SELECT a22 FROM t1 LEFT JOIN t2 ON a11 = a21 WHERE a11 IN (1);
+SELECT a22 FROM t1 LEFT JOIN t2 ON a11 = a21 - a21 WHERE a11 IN (1);
+SELECT a22 FROM t2 RIGHT JOIN t1 ON a21 = a11 WHERE a11 IN (1);
+SELECT a22 FROM t2 RIGHT JOIN t1 ON a21 = a11 WHERE a11 IN (1,2);
+SELECT a22, a32 FROM t1 LEFT JOIN t2 ON a11 = a21 RIGHT JOIN t3 on a21 = a31 WHERE a11 IN (1);
+SELECT a22, a32 FROM t1 LEFT JOIN t2 ON a11 = a21 - a21 RIGHT JOIN t3 on a21 = a31 - a31 WHERE a11 IN (1);
+SELECT a22, a32 FROM t1 LEFT JOIN t2 ON a11 = a21 RIGHT JOIN t3 on a21 = a31 WHERE a11 IN (1,2);
+SELECT a22 FROM t2 LEFT JOIN t1 ON a11 = a21 WHERE a11 IN (1);
+SELECT a22 FROM t2 LEFT JOIN t1 ON a11 = a21 - a21 WHERE a11 IN (1);
+SELECT a22 FROM t1 RIGHT JOIN t2 ON a21 = a11 WHERE a11 IN (1);
+SELECT a22 FROM t1 RIGHT JOIN t2 ON a21 = a11 WHERE a11 IN (1,2);
+
+--connection master_1
+DROP DATABASE IF EXISTS auto_test_local;
+--connection child2_1
+DROP DATABASE IF EXISTS auto_test_remote;
+
+--disable_query_log
+--disable_result_log
+--source ../t/test_deinit.inc
+--enable_query_log
+--enable_result_log

storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test

+--echo #
+--echo # MDEV-29163 Server crash with SIGSEGV or dynamic-stack-buffer-overflow in spider_db_mbase_util::append_table
+--echo #
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+evalp CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (b INT);
+CREATE TABLE t3 (c INT, PRIMARY KEY(c));
+
+CREATE TABLE t1_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't1'";
+CREATE TABLE t2_spider (b INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't2'";
+CREATE TABLE t3_spider (c INT, PRIMARY KEY(c)) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't3'";
+
+SELECT t1_spider.* FROM t1_spider LEFT JOIN t2_spider LEFT JOIN t3_spider ON b = c ON a = b;
+
+# Cleanup
+DROP TABLE t1_spider, t2_spider, t3_spider, t1, t2, t3;
+drop server s;
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--echo #
+--echo # end of test mdev_29163
+--echo #

storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test

+--echo #
+--echo # MDEV-30392 Syntax error upon query with subquery from Spider table
+--echo #
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 'srv', table 't1'";
+
+SELECT a FROM t2 WHERE a IN ( SELECT a FROM t2 );
+
+# Cleanup
+
+DROP TABLE t1, t2;
+DROP SERVER srv;
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--echo #
+--echo # end of test mdev_30392
+--echo #

storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test

+--echo #
+--echo # MDEV-31645 Spider doesn't recognize semi JOIN
+--echo #
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+
+CREATE TABLE t1 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a));
+CREATE TABLE t2 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)) ENGINE=SPIDER COMMENT='srv "srv", WRAPPER "mysql", TABLE "t1"';
+
+SELECT * FROM t2 WHERE b IN (SELECT b FROM t2 WHERE a > 10);
+SELECT * FROM t2
+WHERE A BETWEEN 0 AND 10 AND B IN(SELECT B FROM t2 WHERE A BETWEEN 11 AND 20);
+drop table t1, t2;
+drop server srv;
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--echo #
+--echo # end of test mdev_31645
+--echo #

storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test

+--echo #
+--echo # Test joining a spider table with a non-spider table
+--echo #
+
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+
+evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+create table t1 (c int);
+create table t2 (d int);
+insert into t2 values (1), (2);
+create table t3 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"';
+insert into t3 values (2), (3);
+select c from t3 join t2 on c = d;
+drop table t1, t2, t3;
+drop server srv;
+
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--echo #
+--echo # end of test spider_join_with_non_spider
+--echo #

storage/spider/mysql-test/spider/bugfix/t/subquery.test

+--echo #
+--echo # Test spider select with subqueries
+--echo #
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+create table t1 (c1 int);
+create table t2 (c2 int);
+
+insert into t1 values (1), (2);
+insert into t2 values (0), (1), (2);
+
+create table t1s (c1 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"';
+create table t2s (c2 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
+
+select c1 from t1s, (select c2 from t2s where c2 > 0) t where c1 + 1 = c2;
+
+drop table t1, t2, t1s, t2s;
+drop server srv;
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--echo #
+--echo # end of test subquery
+--echo #

storage/spider/mysql-test/spider/r/direct_join.result

@@ -167,7 +167,7 @@ connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
 select `id`,`hr_status`,`region_code`,`region` from `auto_test_remote`.`tbl_person` where `id` = '24FC3F0A5119432BAE13DD65AABAA39C' and `region` = 510411
-select count(0) `count(0)` from `auto_test_remote`.`tbl_ncd_cm_person` t0 where ((t0.`person_id` = '24FC3F0A5119432BAE13DD65AABAA39C') and (t0.`diseaseKind_id` = '52A0328740914BCE86ED10A4D2521816'))
+select count(0) `count(0)` from (select 1) t0 join `auto_test_remote`.`tbl_ncd_cm_person` t1 where ((t1.`person_id` = '24FC3F0A5119432BAE13DD65AABAA39C') and (t1.`diseaseKind_id` = '52A0328740914BCE86ED10A4D2521816'))
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT * FROM tbl_person;
 id	hr_status	region_code	region

storage/spider/mysql-test/spider/r/direct_left_right_join_nullable.result

@@ -87,7 +87,7 @@ NULL	NULL	NULL	3
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_auto_inc` t2 join `auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r` t0) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null) and (t3.`b` is not null)) where 1 order by t3.`a` desc
+select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r` t0 join `auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r_auto_inc` t2) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null) and (t3.`b` is not null)) where 1 order by t3.`a` desc
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a;
 a	b	date_format(c, '%Y-%m-%d %H:%i:%s')

storage/spider/mysql-test/spider/r/direct_left_right_left_join_nullable.result

@@ -86,7 +86,7 @@ NULL	c	2000-01-03 00:00:00	3
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r` t0) on ((t1.`c` = t2.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null)) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc
+select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r` t0 join `auto_test_remote`.`ta_r_3` t1) on ((t1.`c` = t2.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null)) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a;
 a	b	date_format(c, '%Y-%m-%d %H:%i:%s')

storage/spider/mysql-test/spider/r/direct_right_join.result

@@ -79,7 +79,7 @@ a	b	c
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select t0.`b` `b`,t0.`a` `a`,t2.`b` `b`,t2.`c` `c` from `auto_test_remote`.`ta_r_int` t2 left join (`auto_test_remote`.`ta_r` t0 join `auto_test_remote`.`ta_r_3` t1) on ((t0.`a` = t2.`a`) and (t1.`a` = t2.`a`)) where 1 order by t0.`b` desc
+select t0.`b` `b`,t0.`a` `a`,t2.`b` `b`,t2.`c` `c` from `auto_test_remote`.`ta_r_int` t2 left join (`auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r` t0) on ((t0.`a` = t2.`a`) and (t1.`a` = t2.`a`)) where 1 order by t0.`b` desc
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a;
 a	b	date_format(c, '%Y-%m-%d %H:%i:%s')

storage/spider/mysql-test/spider/r/direct_right_join_nullable.result

@@ -87,7 +87,7 @@ NULL	c	2000-01-03 00:00:00	3
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join `auto_test_remote`.`ta_r_auto_inc` t2 on ((t2.`b` = t3.`b`) and (t3.`b` is not null)) left join `auto_test_remote`.`ta_r_3` t1 on (t1.`c` = t2.`c`) left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null)) where 1 order by t3.`a` desc
+select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null))) on (t1.`c` = t2.`c`)) on ((t2.`b` = t3.`b`) and (t3.`b` is not null)) where 1 order by t3.`a` desc
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a;
 a	b	date_format(c, '%Y-%m-%d %H:%i:%s')

storage/spider/mysql-test/spider/r/direct_right_left_join_nullable.result

@@ -86,7 +86,7 @@ NULL	c	2000-01-03 00:00:00	3
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join `auto_test_remote`.`ta_r_3` t1 on (t1.`c` = t2.`c`) left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null)) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc
+select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null))) on (t1.`c` = t2.`c`) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a;
 a	b	date_format(c, '%Y-%m-%d %H:%i:%s')

storage/spider/mysql-test/spider/r/direct_right_left_right_join_nullable.result

@@ -87,7 +87,7 @@ NULL	c	2000-01-03 00:00:00	3
 connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
-select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_auto_inc` t2 join `auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null))) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`) and (t3.`b` is not null)) where 1 order by t3.`a` desc
+select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null)) join `auto_test_remote`.`ta_r_auto_inc` t2) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`) and (t3.`b` is not null)) where 1 order by t3.`a` desc
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a;
 a	b	date_format(c, '%Y-%m-%d %H:%i:%s')

storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result

@@ -86,8 +86,10 @@ SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
 argument
 select `value` from `auto_test_remote2`.`tbl_a` where `value` = 5
 select `value2` from `auto_test_remote2`.`tbl_b` where `value2` = 5
+select sum('5') `sum(a.value)`,count('5') `count(b.value2)` from (select 1) t0 join (select 1) t1
 select `value` from `auto_test_remote2`.`tbl_a` where `value` = 5
 select `value2` from `auto_test_remote2`.`tbl_b` where `value2` = 5
+select sum('5') `sum(a.value)`,count('5') `count(b.value2)` from (select 1) t0 join (select 1) t1
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
 SELECT value FROM tbl_a ORDER BY value;
 value

storage/spider/mysql-test/spider/regression/e1121/r/direct_join_by_pkey_key.result

@@ -46,7 +46,7 @@ connection child2_1;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%`tbl_a`%' ;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%`tbl_b`%';
 argument
-select t0.`val` `val`,t0.`akey` `akey` from `auto_test_remote`.`tbl_a` t0 where (t0.`akey` = '4')
+select t0.`val` `val`,t0.`akey` `akey` from `auto_test_remote`.`tbl_a` t0 join (select 1) t1 where (t0.`akey` = '4')
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%`tbl_a`%' ;
 SELECT argument FROM mysql.general_log WHERE argument LIKE '%`tbl_b`%'
 argument

storage/spider/spd_db_mysql.cc

@@ -6807,440 +6807,249 @@ int spider_db_mbase_util::append_escaped_util(
 }
 
 #ifdef SPIDER_HAS_GROUP_BY_HANDLER
-int spider_db_mbase_util::append_table(
-  ha_spider *spider,
-  spider_fields *fields,
+/*
+  Walk a TABLE_LIST, or format it to a string and append it.
+
+  If str is NULL, walk the nested join (if any) to determine whether
+  to create a group by handler. Otherwise, format the TABLE_LIST to a
+  string and append it to str.
+
+  Adapted from TABLE_LIST::print().
+*/
+int spider_db_mbase_util::append_table_list(spider_fields *fields,
                                             spider_string *str,
-  TABLE_LIST *table_list,
-  TABLE_LIST **used_table_list,
-  uint *current_pos,
-  TABLE_LIST **cond_table_list_ptr,
-  bool top_down,
-  bool first
-) {
-  int error_num;
-  bool use_cond_table_list = FALSE;
-  spider_mbase_share *db_share;
-  spider_mbase_handler *dbton_hdl;
-  SPIDER_TABLE_HOLDER *table_holder;
-  TABLE_LIST *cond_table_list = *cond_table_list_ptr;
-  ha_spider *spd;
-  DBUG_ENTER("spider_db_mbase_util::append_table");
-  DBUG_PRINT("info",("spider table_list=%p", table_list));
-  DBUG_PRINT("info",("spider table_list->outer_join=%u",
-    table_list->outer_join));
-  DBUG_PRINT("info",("spider table_list->on_expr=%p",
-    table_list->on_expr));
-  DBUG_PRINT("info",("spider table_list->join_using_fields=%p",
-    table_list->join_using_fields));
-  DBUG_PRINT("info",("spider table_list->table=%p",
-    table_list->table));
-  if (!top_down && table_list->embedding)
-  {
-    if ((error_num = append_embedding_tables(spider, fields, str,
-      table_list->embedding, used_table_list, current_pos,
-      cond_table_list_ptr)))
-      DBUG_RETURN(error_num);
-  } else if (!table_list->table)
-  {
-    if ((error_num = append_tables_top_down(spider, fields, str, table_list,
-      used_table_list, current_pos, cond_table_list_ptr)))
-      DBUG_RETURN(error_num);
-  } else {
-    if (
-      table_list->outer_join ||
-      table_list->on_expr ||
-      table_list->join_using_fields
-    ) {
-      DBUG_PRINT("info",("spider use table_list"));
-      if (table_list->outer_join & JOIN_TYPE_LEFT)
-      {
-        if (str)
-        {
-          if (str->reserve(SPIDER_SQL_LEFT_JOIN_LEN))
-          {
-            DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-          }
-          str->q_append(SPIDER_SQL_LEFT_JOIN_STR, SPIDER_SQL_LEFT_JOIN_LEN);
+                                            TABLE_LIST *table,
+                                            table_map *upper_usable_tables,
+                                            table_map eliminated_tables)
+{
+  DBUG_ENTER("spider_db_mbase_util::append_table_list");
+  /* Eliminated tables were removed from append_join(). */
+  DBUG_ASSERT(!is_eliminated_table(eliminated_tables, table));
+  if (!str)                     /* First pass (GBH creation) */
+  {
+    DBUG_ASSERT(upper_usable_tables);
+    if (table->nested_join)
+      DBUG_RETURN(append_join(fields, str,
+                              &table->nested_join->join_list,
+                              upper_usable_tables, eliminated_tables));
+    /* jtbm is a kind of semi join, and TABLE_LIST::print() adds an
+    extra " <materialize> " annotation. */
+    if (table->jtbm_subselect)
+      DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
+    /* TODO: These conditions are printed in a different way in
+    TABLE_LIST::print(), but they do not seem to occur very often.
+    Let's not worry about them now. */
+    if (table->view_name.str || table->derived)
+      DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
+    *upper_usable_tables |= table->table->map;
+    DBUG_RETURN(0);
   }
-      } else {
-        if (str)
-        {
-          if (str->reserve(SPIDER_SQL_JOIN_LEN))
+  /* Second pass (query execution) */
+  DBUG_ASSERT(!upper_usable_tables);
+  if (table->nested_join)
   {
+    if (str->append("("))
       DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-          }
-          str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN);
-        }
-      }
-    } else if (
-      cond_table_list &&
-      (
-        cond_table_list->outer_join ||
-        cond_table_list->on_expr ||
-        cond_table_list->join_using_fields
-      )
-    ) {
-      DBUG_PRINT("info",("spider use cond_table_list"));
-      if (cond_table_list->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT))
-      {
-        if (str)
-        {
-          if (str->reserve(SPIDER_SQL_LEFT_JOIN_LEN))
-          {
+    if (int error_num= append_join(
+          fields, str, &table->nested_join->join_list,
+          upper_usable_tables, eliminated_tables))
+      DBUG_RETURN(error_num);
+    if (str->append(")"))
       DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    DBUG_RETURN(0);
   }
-          str->q_append(SPIDER_SQL_LEFT_JOIN_STR, SPIDER_SQL_LEFT_JOIN_LEN);
-        }
-      } else {
-        if (str)
-        {
-          if (str->reserve(SPIDER_SQL_JOIN_LEN))
+  /* These have been ruled out during the creation of the group by
+  handler, see above. */
+  DBUG_ASSERT(!table->jtbm_subselect);
+  DBUG_ASSERT(!table->view_name.str);
+  DBUG_ASSERT(!table->derived);
+  /* We have a "normal" table. Print it and append to str. */
+  SPIDER_TABLE_HOLDER *table_holder = fields->get_table_holder(table->table);
+  ha_spider *spd = table_holder->spider;
+  spider_mbase_share *db_share =
+    (spider_mbase_share *) spd->share->dbton_share[dbton_id];
+  spider_mbase_handler *dbton_hdl =
+    (spider_mbase_handler *) spd->dbton_handler[dbton_id];
+  if (table->table->const_table)
   {
+    if (str->append(STRING_WITH_LEN("(select 1)")))
       DBUG_RETURN(HA_ERR_OUT_OF_MEM);
   }
-          str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN);
-        }
-      }
-      use_cond_table_list = TRUE;
-    } else if (*current_pos > 0 && !first)
-    {
-      DBUG_PRINT("info",("spider no condition"));
-      if (str)
-      {
-        if (str->reserve(SPIDER_SQL_JOIN_LEN))
-        {
+  else
+    if (int error_num= db_share->append_table_name(
+          str, spd->conn_link_idx[dbton_hdl->first_link_idx]))
+      DBUG_RETURN(error_num);
+  if (str->append(" ") ||
+      str->append(table_holder->alias->ptr(),
+                  /* Don't append the trailing dot */
+                  table_holder->alias->length() - 1))
     DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN);
-      }
-    }
-
-    if (str)
-    {
-      table_holder = fields->get_table_holder(table_list->table);
-      spd = table_holder->spider;
-      db_share = (spider_mbase_share *)
-        spd->share->dbton_share[dbton_id];
-      dbton_hdl = (spider_mbase_handler *)
-        spd->dbton_handler[dbton_id];
+  DBUG_RETURN(0);
+}
 
-      dbton_hdl->table_name_pos = str->length();
+/*
+  Walk an array of TABLE_LIST's, or format it to a string and append it.
 
-      if (str->reserve(
-        db_share->db_nm_max_length +
-        SPIDER_SQL_DOT_LEN + /* SPIDER_SQL_NAME_QUOTE_LEN */ 4 +
-        db_share->table_nm_max_length + SPIDER_SQL_SPACE_LEN +
-        table_holder->alias->length() - SPIDER_SQL_DOT_LEN
-      )) {
-        DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-      }
+  If str is NULL, walk each TABLE_LIST to determine whether to create
+  a group by handler. Otherwise, format the TABLE_LISTs to a string
+  and append it to str.
 
-      if ((error_num = db_share->append_table_name_with_adjusting(str,
-        spd->conn_link_idx[dbton_hdl->first_link_idx])))
+  Adapted from print_table_array().
+*/
+int spider_db_mbase_util::append_table_array(spider_fields *fields,
+                                             spider_string *str,
+                                             TABLE_LIST **table,
+                                             TABLE_LIST **end,
+                                             table_map *upper_usable_tables,
+                                             table_map eliminated_tables)
+{
+  DBUG_ENTER("spider_db_mbase_util::append_table_array");
+  if (str)
   {
+    DBUG_ASSERT(!upper_usable_tables);
+    if (int error_num= append_table_list(fields, str, *table, NULL,
+                                         eliminated_tables))
       DBUG_RETURN(error_num);
-      }
-      str->q_append(SPIDER_SQL_SPACE_STR, SPIDER_SQL_SPACE_LEN);
-      str->q_append(table_holder->alias->ptr(),
-        table_holder->alias->length() - SPIDER_SQL_DOT_LEN);
-    }
-    used_table_list[(*current_pos)++] = table_list;
 
-    if (str)
-    {
-      List<String> *join_using_fields = table_list->join_using_fields;
-      if (!join_using_fields && cond_table_list)
+    for (TABLE_LIST **tbl= table + 1; tbl < end; tbl++)
     {
-        join_using_fields = cond_table_list->join_using_fields;
-      }
+      TABLE_LIST *curr= *tbl;
 
-      if (join_using_fields)
-      {
-        if (str->reserve(SPIDER_SQL_USING_LEN + SPIDER_SQL_OPEN_PAREN_LEN))
+      /* JOIN_TYPE_OUTER is just a marker unrelated to real join */
+      if (curr->outer_join & (JOIN_TYPE_LEFT|JOIN_TYPE_RIGHT))
       {
+        /* MySQL converts right to left joins */
+        if (str->append(STRING_WITH_LEN(" left join ")))
           DBUG_RETURN(HA_ERR_OUT_OF_MEM);
       }
-        str->q_append(SPIDER_SQL_USING_STR, SPIDER_SQL_USING_LEN);
-        str->q_append(SPIDER_SQL_OPEN_PAREN_STR,
-          SPIDER_SQL_OPEN_PAREN_LEN);
-        List_iterator_fast<String> it2(*join_using_fields);
-        String *ptr;
-        while ((ptr = it2++))
-        {
-          if (str->reserve(ptr->length() + SPIDER_SQL_COMMA_LEN))
+      else if (curr->straight)
       {
+        if (str->append(STRING_WITH_LEN(" straight_join ")))
           DBUG_RETURN(HA_ERR_OUT_OF_MEM);
       }
-          str->q_append(ptr->ptr(), ptr->length());
-          str->q_append(SPIDER_SQL_COMMA_STR, SPIDER_SQL_COMMA_LEN);
-        }
-        str->length(str->length() - SPIDER_SQL_COMMA_LEN);
-        if (str->reserve(SPIDER_SQL_CLOSE_PAREN_LEN))
-        {
+      /* semi join should already have been ruled out during the
+      creation of the group by handler. */
+      else if (curr->sj_inner_tables)
+        DBUG_ASSERT(0);
+      else
+        if (str->append(STRING_WITH_LEN(" join ")))
           DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_CLOSE_PAREN_STR,
-          SPIDER_SQL_CLOSE_PAREN_LEN);
-      }
-    }
-
-    Item *on_expr = table_list->on_expr;
-    if (!on_expr && cond_table_list)
-    {
-      on_expr = cond_table_list->on_expr;
-    }
 
-    if (on_expr)
-    {
-      if (str)
-      {
-        if (str->reserve(SPIDER_SQL_ON_LEN))
-        {
-          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_ON_STR, SPIDER_SQL_ON_LEN);
-      }
-      if ((error_num = spider_db_print_item_type(on_expr, NULL,
-        spider, str, NULL, 0, dbton_id, TRUE, fields)))
-      {
+      if (int error_num= append_table_list(fields, str, curr, NULL,
+                                           eliminated_tables))
         DBUG_RETURN(error_num);
-      }
-    }
-
-    if (use_cond_table_list)
-    {
-      (*cond_table_list_ptr) = NULL;
-      DBUG_PRINT("info",("spider cond_table_list=%p", (*cond_table_list_ptr)));
-    }
-  }
-  DBUG_RETURN(0);
-}
 
-int spider_db_mbase_util::append_tables_top_down(
-  ha_spider *spider,
-  spider_fields *fields,
-  spider_string *str,
-  TABLE_LIST *table_list,
-  TABLE_LIST **used_table_list,
-  uint *current_pos,
-  TABLE_LIST **cond_table_list_ptr
-) {
-  int error_num;
-  uint outer_join_backup;
-  TABLE_LIST *cur_table_list, *prev_table_list = NULL, *cond_table_list = NULL;
-  bool first = TRUE;
-  DBUG_ENTER("spider_db_mbase_util::append_tables_top_down");
-  DBUG_PRINT("info",("spider this=%p", this));
-  if (
-    table_list->outer_join ||
-    table_list->on_expr ||
-    table_list->join_using_fields
-  ) {
-    DBUG_ASSERT(!(*cond_table_list_ptr));
-    (*cond_table_list_ptr) = table_list;
-    DBUG_PRINT("info",("spider cond_table_list=%p", table_list));
-  }
-  List_iterator_fast<TABLE_LIST> it1(table_list->nested_join->join_list);
-  cur_table_list = it1++;
-  if (cur_table_list->outer_join & JOIN_TYPE_RIGHT)
-  {
-    first = FALSE;
-    prev_table_list = cur_table_list;
-    cur_table_list = it1++;
-  } else if (*cond_table_list_ptr)
-  {
-    first = TRUE;
-    cond_table_list = (*cond_table_list_ptr);
-    (*cond_table_list_ptr) = NULL;
-    if (cond_table_list->outer_join & JOIN_TYPE_LEFT)
-    {
-      if (str)
-      {
-        if (str->reserve(SPIDER_SQL_LEFT_JOIN_LEN + SPIDER_SQL_OPEN_PAREN_LEN))
+      if (curr->on_expr)
       {
+        if (str->append(STRING_WITH_LEN(" on ")))
           DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_LEFT_JOIN_STR, SPIDER_SQL_LEFT_JOIN_LEN);
-        str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN);
-      }
-    } else {
-      if (str)
-      {
-        if (str->reserve(SPIDER_SQL_JOIN_LEN + SPIDER_SQL_OPEN_PAREN_LEN))
-        {
-          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN);
-        str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN);
+        if (int error_num=
+            spider_db_print_item_type(curr->on_expr, NULL,
+                                      fields->get_first_table_holder()->spider,
+                                      str, NULL, 0, dbton_id, TRUE, fields))
+          DBUG_RETURN(error_num);
       }
     }
   }
-
-  do {
-    if (cur_table_list->outer_join & JOIN_TYPE_RIGHT)
+  else                        /* str == NULL */
   {
-      prev_table_list = cur_table_list;
-    } else {
-      if ((error_num = append_table(spider, fields, str, cur_table_list,
-        used_table_list, current_pos, cond_table_list_ptr, TRUE, first)))
+    table_map usable_tables= 0;
+    if (int error_num= append_table_list(fields, str, *table,
+                                         &usable_tables, eliminated_tables))
       DBUG_RETURN(error_num);
-      first = FALSE;
-      if (prev_table_list)
+    for (TABLE_LIST **tbl= table + 1; tbl < end; tbl++)
     {
-        outer_join_backup = prev_table_list->outer_join;
-        prev_table_list->outer_join = JOIN_TYPE_LEFT;
-        if ((error_num = append_table(spider, fields, str, prev_table_list,
-          used_table_list, current_pos, cond_table_list_ptr, TRUE, FALSE)))
+      TABLE_LIST *curr= *tbl;
+      /* semi join is an "internal" join and is unsupported. */
+      if (curr->sj_inner_tables)
+        DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
+      if (int error_num= append_table_list(fields, str, curr,
+                                           &usable_tables, eliminated_tables))
+        DBUG_RETURN(error_num);
+      if (curr->on_expr)
       {
-          prev_table_list->outer_join = outer_join_backup;
+        /* The join refers to fields outside of the current context,
+        and cannot be handled by a group by handler. */
+        if ((curr->on_expr->used_tables() & usable_tables) !=
+            curr->on_expr->used_tables())
+          DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
+        if (int error_num=
+            spider_db_print_item_type(curr->on_expr, NULL,
+                                      fields->get_first_table_holder()->spider,
+                                      str, NULL, 0, dbton_id, TRUE, fields))
           DBUG_RETURN(error_num);
       }
-        prev_table_list->outer_join = outer_join_backup;
-        prev_table_list = NULL;
     }
+    /* Update usable tables in the outer context. */
+    if (upper_usable_tables)
+      *upper_usable_tables |= usable_tables;
   }
-  } while ((cur_table_list = it1++));
+  DBUG_RETURN(0);
+}
 
-  if (cond_table_list)
-  {
-    if (str)
-    {
-      if (str->reserve(SPIDER_SQL_CLOSE_PAREN_LEN))
-      {
-        DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-      }
-      str->q_append(SPIDER_SQL_CLOSE_PAREN_STR,
-        SPIDER_SQL_CLOSE_PAREN_LEN);
+/*
+  Walk a join, or format it to a string and append the string.
 
-      List<String> *join_using_fields = cond_table_list->join_using_fields;
-      if (join_using_fields)
-      {
-        if (str->reserve(SPIDER_SQL_USING_LEN + SPIDER_SQL_OPEN_PAREN_LEN))
-        {
-          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_USING_STR, SPIDER_SQL_USING_LEN);
-        str->q_append(SPIDER_SQL_OPEN_PAREN_STR,
-          SPIDER_SQL_OPEN_PAREN_LEN);
-        List_iterator_fast<String> it2(*join_using_fields);
-        String *ptr;
-        while ((ptr = it2++))
-        {
-          if (str->reserve(ptr->length() + SPIDER_SQL_COMMA_LEN))
+  Skip all eliminated tables.
+
+  If str is NULL, walk the tables to determine whether to create a
+  group by handler. Otherwise, format the join to a string and append
+  it to str.
+
+  Adapted from print_join().
+*/
+int spider_db_mbase_util::append_join(spider_fields *fields,
+                                      spider_string *str,
+                                      List<TABLE_LIST> *tables,
+                                      table_map *upper_usable_tables,
+                                      table_map eliminated_tables)
+{
+  /* List is reversed => we should reverse it before using */
+  List_iterator_fast<TABLE_LIST> ti(*tables);
+  TABLE_LIST **table;
+  THD *thd= fields->get_first_table_holder()->spider->trx->thd;
+  DBUG_ENTER("spider_db_mbase_util::append_join");
+
+  size_t tables_to_print= 0;
+
+  for (TABLE_LIST *t= ti++; t ; t= ti++)
   {
-            DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    /* optimized_away implies const_table */
+    DBUG_ASSERT(!t->optimized_away || t->table->const_table);
+    if (!is_eliminated_table(eliminated_tables, t))
+      tables_to_print++;
   }
-          str->q_append(ptr->ptr(), ptr->length());
-          str->q_append(SPIDER_SQL_COMMA_STR, SPIDER_SQL_COMMA_LEN);
-        }
-        str->length(str->length() - SPIDER_SQL_COMMA_LEN);
-        if (str->reserve(SPIDER_SQL_CLOSE_PAREN_LEN))
+  if (tables_to_print == 0)
   {
+    if (str && str->append(STRING_WITH_LEN("dual")))
       DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    DBUG_RETURN(0);
   }
-        str->q_append(SPIDER_SQL_CLOSE_PAREN_STR,
-          SPIDER_SQL_CLOSE_PAREN_LEN);
-      }
-    }
+  ti.rewind();
 
-    Item *on_expr = cond_table_list->on_expr;
-    if (on_expr)
-    {
-      if (str)
-      {
-        if (str->reserve(SPIDER_SQL_ON_LEN))
-        {
+  if (!(table= static_cast<TABLE_LIST **>(thd->alloc(sizeof(TABLE_LIST*) *
+                                                     tables_to_print))))
     DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-        }
-        str->q_append(SPIDER_SQL_ON_STR, SPIDER_SQL_ON_LEN);
-      }
-      if ((error_num = spider_db_print_item_type(on_expr, NULL,
-        spider, str, NULL, 0, dbton_id, TRUE, fields)))
-      {
-        DBUG_RETURN(error_num);
-      }
-    }
-  }
-  DBUG_RETURN(0);
-}
 
-int spider_db_mbase_util::append_tables_top_down_check(
-  TABLE_LIST *table_list,
-  TABLE_LIST **used_table_list,
-  uint *current_pos
-) {
-  int error_num;
-  TABLE_LIST *cur_table_list;
-  DBUG_ENTER("spider_db_mbase_util::append_tables_top_down_check");
-  DBUG_PRINT("info",("spider this=%p", this));
-  List_iterator_fast<TABLE_LIST> it1(table_list->nested_join->join_list);
-  while ((cur_table_list = it1++))
-  {
-    if (!cur_table_list->table)
+  TABLE_LIST *tmp, **t= table + (tables_to_print - 1);
+  while ((tmp= ti++))
+    if (!is_eliminated_table(eliminated_tables, tmp))
+      *t--= tmp;
+
+  DBUG_ASSERT(tables->elements >= 1);
+  if ((*table)->sj_inner_tables)
   {
-      if ((error_num = append_tables_top_down_check(
-        cur_table_list, used_table_list, current_pos)))
-        DBUG_RETURN(error_num);
-    } else {
-      used_table_list[(*current_pos)++] = cur_table_list;
-    }
+    /* Semi join is not supported. */
+    if (!str)
+      DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
+    /* Semi join should have been skipped in the first pass. */
+    else
+      DBUG_ASSERT(0);
   }
-  DBUG_RETURN(0);
-}
-
-int spider_db_mbase_util::append_embedding_tables(
-  ha_spider *spider,
-  spider_fields *fields,
-  spider_string *str,
-  TABLE_LIST *table_list,
-  TABLE_LIST **used_table_list,
-  uint *current_pos,
-  TABLE_LIST **cond_table_list_ptr
-) {
-  int error_num;
-  TABLE_LIST *embedding = table_list->embedding;
-  DBUG_ENTER("spider_db_mbase_util::append_embedding_tables");
-  DBUG_PRINT("info",("spider this=%p", this));
-  if (embedding)
-  {
-    DBUG_PRINT("info",("spider embedding=%p", embedding));
-    DBUG_PRINT("info",("spider embedding->outer_join=%u",
-      embedding->outer_join));
-    DBUG_PRINT("info",("spider embedding->on_expr=%p",
-      embedding->on_expr));
-    DBUG_PRINT("info",("spider embedding->join_using_fields=%p",
-      embedding->join_using_fields));
-    DBUG_PRINT("info",("spider embedding->table=%p",
-      embedding->table));
-    if ((error_num = append_embedding_tables(spider, fields, str, embedding,
-      used_table_list, current_pos, cond_table_list_ptr)))
+  int error_num= append_table_array(
+    fields, str, table, table + tables_to_print, upper_usable_tables,
+    eliminated_tables);
   DBUG_RETURN(error_num);
-  } else {
-    DBUG_PRINT("info",("spider table_list=%p", table_list));
-    DBUG_PRINT("info",("spider table_list->outer_join=%u",
-      table_list->outer_join));
-    DBUG_PRINT("info",("spider table_list->on_expr=%p",
-      table_list->on_expr));
-    DBUG_PRINT("info",("spider table_list->join_using_fields=%p",
-      table_list->join_using_fields));
-    DBUG_PRINT("info",("spider table_list->table=%p",
-      table_list->table));
-    if (table_list->outer_join & JOIN_TYPE_RIGHT)
-    {
-      if ((error_num = append_tables_top_down_check(table_list,
-        used_table_list, current_pos)))
-        DBUG_RETURN(error_num);
-      DBUG_ASSERT(!(*cond_table_list_ptr));
-      (*cond_table_list_ptr) = table_list;
-      DBUG_PRINT("info",("spider cond_table_list=%p", table_list));
-    } else {
-      if ((error_num = append_tables_top_down(spider, fields, str, table_list,
-        used_table_list, current_pos, cond_table_list_ptr)))
-        DBUG_RETURN(error_num);
-    }
-  }
-  DBUG_RETURN(0);
 }
 
 int spider_db_mbase_util::append_from_and_tables(
@@ -7249,77 +7058,17 @@ int spider_db_mbase_util::append_from_and_tables(
   spider_string *str,
   TABLE_LIST *table_list,
   uint table_count
-) {
-  int error_num;
-  uint current_pos = 0, roop_count, backup_pos, outer_join_backup;
-  TABLE *table;
-  TABLE_LIST **used_table_list, *prev_table_list = NULL,
-    *cond_table_list = NULL;
+)
+{
   DBUG_ENTER("spider_db_mbase_util::append_from_and_tables");
-  DBUG_PRINT("info",("spider this=%p", this));
-  used_table_list = (TABLE_LIST **)
-    my_alloca(sizeof(TABLE_LIST *) * table_count);
-  if (!used_table_list)
-    DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-
-  if (str)
-  {
-    if (str->reserve(SPIDER_SQL_FROM_LEN))
-    {
-      my_afree(used_table_list);
+  if (str && str->append(" from "))
     DBUG_RETURN(HA_ERR_OUT_OF_MEM);
-    }
-    str->q_append(SPIDER_SQL_FROM_STR, SPIDER_SQL_FROM_LEN);
-  }
-
-  do {
-    table = table_list->table;
-    if (table->const_table)
-      continue;
-
-    for (roop_count = 0; roop_count < current_pos; ++roop_count)
-    {
-      if (used_table_list[roop_count] == table_list)
-        break;
-    }
-    if (roop_count < current_pos)
-      continue;
-
-    if (prev_table_list)
-      current_pos = backup_pos;
-    else
-      backup_pos = current_pos;
-    if ((error_num = append_table(spider, fields, str, table_list, used_table_list,
-      &current_pos, &cond_table_list, FALSE, FALSE)))
-    {
-      my_afree(used_table_list);
-      DBUG_RETURN(error_num);
-    }
-    if (prev_table_list)
-    {
-      outer_join_backup = prev_table_list->outer_join;
-      prev_table_list->outer_join = JOIN_TYPE_LEFT;
-      if ((error_num = append_table(spider, fields, str, prev_table_list,
-        used_table_list, &current_pos, &cond_table_list, FALSE, FALSE)))
-      {
-        prev_table_list->outer_join = outer_join_backup;
-        my_afree(used_table_list);
+  const table_map eliminated_tables= table_list->select_lex->join ?
+    table_list->select_lex->join->eliminated_tables : 0;
+  int error_num = append_join(fields, str, table_list->select_lex->join_list,
+                              NULL, eliminated_tables);
   DBUG_RETURN(error_num);
-      }
-      prev_table_list->outer_join = outer_join_backup;
-      prev_table_list = NULL;
-    }
-    if (cond_table_list && (cond_table_list->outer_join & JOIN_TYPE_RIGHT))
-    {
-      prev_table_list = cond_table_list;
-      cond_table_list = NULL;
-      DBUG_PRINT("info",("spider cond_table_list=%p", cond_table_list));
-    }
-  } while ((table_list = table_list->next_local));
-  my_afree(used_table_list);
-  DBUG_RETURN(0);
 }
-
 int spider_db_mbase_util::append_where(
   spider_string *str
 ) {

storage/spider/spd_db_mysql.h

@@ -157,40 +157,22 @@ class spider_db_mbase_util: public spider_db_util
     String *from
   ) override;
 #ifdef SPIDER_HAS_GROUP_BY_HANDLER
-  int append_table(
-    ha_spider *spider,
-    spider_fields *fields,
-    spider_string *str,
-    TABLE_LIST *table_list,
-    TABLE_LIST **used_table_list,
-    uint *current_pos,
-    TABLE_LIST **cond_table_list_ptr,
-    bool top_down,
-    bool first
-  );
-  int append_tables_top_down(
-    ha_spider *spider,
-    spider_fields *fields,
-    spider_string *str,
-    TABLE_LIST *table_list,
-    TABLE_LIST **used_table_list,
-    uint *current_pos,
-    TABLE_LIST **cond_table_list_ptr
-  );
   int append_tables_top_down_check(
     TABLE_LIST *table_list,
     TABLE_LIST **used_table_list,
     uint *current_pos
   );
-  int append_embedding_tables(
-    ha_spider *spider,
-    spider_fields *fields,
-    spider_string *str,
-    TABLE_LIST *table_list,
-    TABLE_LIST **used_table_list,
-    uint *current_pos,
-    TABLE_LIST **cond_table_list_ptr
-  );
+  int append_table_list(spider_fields *fields,
+                        spider_string *str, TABLE_LIST *table,
+                        table_map *upper_usable_tables,
+                        table_map eliminated_tables);
+  int append_table_array(spider_fields *fields,
+                         spider_string *str, TABLE_LIST **table,
+                         TABLE_LIST **end, table_map *upper_usable_tables,
+                         table_map eliminated_tables);
+  int append_join(spider_fields *fields, spider_string *str,
+                  List<TABLE_LIST> *tables, table_map *upper_usable_tables,
+                  table_map eliminated_tables);
   int append_from_and_tables(
     ha_spider *spider,
     spider_fields *fields,

storage/spider/spd_group_by_handler.cc

@@ -25,6 +25,7 @@
 #include "probes_mysql.h"
 #include "sql_class.h"
 #include "sql_partition.h"
+#include "sql_select.h"
 #include "ha_partition.h"
 #endif
 #include "sql_common.h"
@@ -1282,14 +1283,16 @@ static int spider_send_query(
   DBUG_RETURN(0);
 }
 
+/*
+ Prepare and send query to data nodes and store the query results.
+*/
 int spider_group_by_handler::init_scan()
 {
   int error_num;
   DBUG_ENTER("spider_group_by_handler::init_scan");
   store_error = 0;
 #ifndef DBUG_OFF
-  Field **field;
-  for (field = table->field; *field; field++)
+  for (Field **field = table->field; *field; field++)
     DBUG_PRINT("info",("spider field_name=%s", SPIDER_field_name_str(*field)));
 #endif
 
@@ -1439,8 +1442,6 @@ group_by_handler *spider_create_group_by_handler(
   from = query->from;
   do {
     DBUG_PRINT("info",("spider from=%p", from));
-    if (from->table->const_table)
-      continue;
     ++table_count;
     if (from->table->part_info)
     {
@@ -1468,15 +1469,6 @@ group_by_handler *spider_create_group_by_handler(
 
   table_idx = 0;
   from = query->from;
-  while (from && from->table->const_table)
-  {
-    from = from->next_local;
-  }
-  if (!from)
-  {
-    /* all tables are const_table */
-    goto skip_free_table_holder;
-  }
 #if defined(PARTITION_HAS_GET_CHILD_HANDLERS)
   if (from->table->part_info)
   {
@@ -1512,8 +1504,6 @@ group_by_handler *spider_create_group_by_handler(
   }
   while ((from = from->next_local))
   {
-    if (from->table->const_table)
-      continue;
 #if defined(PARTITION_HAS_GET_CHILD_HANDLERS)
     if (from->table->part_info)
     {
@@ -1556,8 +1546,6 @@ group_by_handler *spider_create_group_by_handler(
 
   from = query->from;
   do {
-    if (from->table->const_table)
-      continue;
 #if defined(PARTITION_HAS_GET_CHILD_HANDLERS)
     if (from->table->part_info)
     {
@@ -1701,10 +1689,6 @@ group_by_handler *spider_create_group_by_handler(
     goto skip_free_table_holder;
 
   from = query->from;
-  while (from->table->const_table)
-  {
-    from = from->next_local;
-  }
 #if defined(PARTITION_HAS_GET_CHILD_HANDLERS)
   if (from->table->part_info)
   {
@@ -1777,8 +1761,6 @@ group_by_handler *spider_create_group_by_handler(
 
   while ((from = from->next_local))
   {
-    if (from->table->const_table)
-      continue;
     fields->clear_conn_holder_from_conn();
 
 #if defined(PARTITION_HAS_GET_CHILD_HANDLERS)