MDEV-28856 Add remaining Spider table options

MDEV-27106 added REMOTE_TABLE, REMOTE_DATABASE, REMOTE_SERVER spider
table options. In this commit, we add all remaining options for table
params that are not marked to be deprecated.

All these options are parsed as strings from sql statements and have
string values at the sql level, so that we can determine whether it is
specified by checking its nullness.

The string values are further parsed by Spider into their actual types
in the SPIDER_SHARE, including string list, bounded nonnegative int,
bounded nonnegative int list, nonnegative longlong, boolean, and key
hints. Except for string lists, all other types are validated during
this parsing process.

Most of the options are backward compatible, i.e. they accept any
values that is accepted by there corresponding param parser. The only
exception is the index hint IDX which corresponds to the idxNNN param
name. For example,

'idx000 "f PRIMARY", idx001 "u k1"'

translates to

IDX="f PRIMARY u k1".

We include a test with all options specified, and tests involving
spider table options of all actual types.

Any table options, if present, will cause comments to be ignored with
a warning. The warning can be disabled by setting a new spider
global/session system variable spider_suppress_comment_ignored_warning
to 1.

Another global/session variable introduced is spider_ignore_comments,
which if set to 1, will cause COMMENT and CONNECTION strings to be
ignored unconditionally, whether or not table options are specified.

sql/sql_yacc.yy

@@ -1333,6 +1333,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize);
         ident
         label_ident
         sp_decl_ident
+        ident_options
         ident_or_empty
         ident_table_alias
         ident_sysvar_name
@@ -1363,6 +1364,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize);
         keyword_cast_type
         keyword_ident
         keyword_label
+        keyword_options
         keyword_set_special_case
         keyword_set_usual_case
         keyword_sp_block_section
@@ -5646,32 +5648,49 @@ create_table_option:
         ;
 
 engine_defined_option:
-          IDENT_sys equal TEXT_STRING_sys
+          ident_options equal TEXT_STRING_sys
           {
             if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH))
               my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str));
             $$= new (thd->mem_root) engine_option_value($1, $3, true);
             MYSQL_YYABORT_UNLESS($$);
           }
-        | IDENT_sys equal ident
+        | ident_options equal ident
           {
             if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH))
               my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str));
             $$= new (thd->mem_root) engine_option_value($1, $3, false);
             MYSQL_YYABORT_UNLESS($$);
           }
-        | IDENT_sys equal real_ulonglong_num
+        | ident_options equal real_ulonglong_num
           {
             $$= new (thd->mem_root) engine_option_value($1, $3, thd->mem_root);
             MYSQL_YYABORT_UNLESS($$);
           }
-        | IDENT_sys equal DEFAULT
+        | ident_options equal DEFAULT
           {
             $$= new (thd->mem_root) engine_option_value($1);
             MYSQL_YYABORT_UNLESS($$);
           }
         ;
 
+ident_options:
+          IDENT_sys
+        | keyword_options
+          {
+            if (unlikely($$.copy_keyword(thd, &$1)))
+              MYSQL_YYABORT;
+          }
+        ;
+
+/*
+  These keywords are fine for option names.
+*/
+keyword_options:
+          READ_ONLY_SYM
+        | WRAPPER_SYM
+        ;
+
 opt_versioning_option:
           /* empty */
         | versioning_option

storage/spider/mysql-test/spider/bugfix/r/mdev_32486.result

+#
+# MDEV-32486 Assertion `!trx->alloc_line_no[id] || trx->alloc_line_no[id] == line_no' failed in spider_alloc_mem_calc
+#
+for master_1
+for child2
+for child3
+CREATE TABLE t (c INT) ENGINE=Spider REMOTE_PORT="1";
+DROP TABLE t;
+CREATE TABLE t (c INT) ENGINE=Spider COMMENT="WRAPPER 'mysql', SERVER 's',MONITORING_KIND '1'";
+ERROR HY000: The foreign server name you are trying to reference does not exist. Data source error:  s
+CREATE TABLE t (c INT) ENGINE=Spider COMMENT="WRAPPER 'mysql',SRV 's',MONITORING_KIND '2'";
+ERROR HY000: The foreign server name you are trying to reference does not exist. Data source error:  s
+CREATE TABLE t (c INT) ENGINE=Spider REMOTE_PORT="1";
+drop table t;
+for master_1
+for child2
+for child3
+#
+# end of test mdev_32486
+#

storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result → storage/spider/mysql-test/spider/bugfix/r/sysvar_params.result

 
-MDEV-31524 Spider variables that double as table params overriding mechanism is buggy
+Testing spider sysvar and table params / options, including default values and overriding mechanisms
 
 for master_1
 for child2
 for child3
 SET @old_spider_read_only_mode = @@session.spider_read_only_mode;
-CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+# Cases where table params/options are not set
 set session spider_read_only_mode = default;
 create table t2 (c int);
-create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 1 */ insert into t1 values (42);
 drop table t1, t2;
 set session spider_read_only_mode = 1;
 create table t2 (c int);
-create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 2 */ insert into t1 values (42);
 ERROR HY000: Table 'test.t1' is read only
 drop table t1, t2;
@@ -21,30 +22,72 @@ set session spider_read_only_mode = -1;
 Warnings:
 Warning	138	The option value -1 (fallback to default) is deprecated and will be removed in a future release
 create table t2 (c int);
-create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 3 */ insert into t1 values (42);
 drop table t1, t2;
+# Cases where table params are set
 SET session spider_read_only_mode = default;
 create table t2 (c int);
-create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 4 */ insert into t1 values (42);
 ERROR HY000: Table 'test.t1' is read only
 drop table t1, t2;
 set session spider_read_only_mode = 1;
 create table t2 (c int);
-create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "0", WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "0", WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 5 */ insert into t1 values (42);
 drop table t1, t2;
 SET session spider_read_only_mode = -1;
 Warnings:
 Warning	138	The option value -1 (fallback to default) is deprecated and will be removed in a future release
 create table t2 (c int);
-create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "",TABLE "t2"';
 /* 6 */ insert into t1 values (42);
 ERROR HY000: Table 'test.t1' is read only
 drop table t1, t2;
-drop server srv_mdev_31524;
+# Cases where table options are set
+SET session spider_read_only_mode = default;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider REMOTE_SERVER=srv REMOTE_TABLE=t2 READ_ONLY=1 WRAPPER=mysql;
+/* 7 */ insert into t1 values (42);
+ERROR HY000: Table 'test.t1' is read only
+drop table t1, t2;
+set session spider_read_only_mode = 1;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=0 REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 8 */ insert into t1 values (42);
+drop table t1, t2;
+SET session spider_read_only_mode = -1;
+Warnings:
+Warning	138	The option value -1 (fallback to default) is deprecated and will be removed in a future release
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=1 REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 9 */ insert into t1 values (42);
+ERROR HY000: Table 'test.t1' is read only
+drop table t1, t2;
+SET session spider_read_only_mode = 0;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=default REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 10 */ insert into t1 values (42);
+drop table t1, t2;
+SET session spider_read_only_mode = 1;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=default REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 11 */ insert into t1 values (42);
+ERROR HY000: Table 'test.t1' is read only
+drop table t1, t2;
+SET session spider_read_only_mode = -1;
+Warnings:
+Warning	138	The option value -1 (fallback to default) is deprecated and will be removed in a future release
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=default REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 12 */ insert into t1 values (42);
+drop table t1, t2;
+drop server srv;
 SET session spider_read_only_mode = @old_spider_read_only_mode;
 for master_1
 for child2
 for child3
+#
+# End of test sysvar_params
+#

storage/spider/mysql-test/spider/bugfix/t/mdev_32486.test

+--echo #
+--echo # MDEV-32486 Assertion `!trx->alloc_line_no[id] || trx->alloc_line_no[id] == line_no' failed in spider_alloc_mem_calc
+--echo #
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+CREATE TABLE t (c INT) ENGINE=Spider REMOTE_PORT="1";
+DROP TABLE t;
+--error ER_FOREIGN_SERVER_DOESNT_EXIST
+CREATE TABLE t (c INT) ENGINE=Spider COMMENT="WRAPPER 'mysql', SERVER 's',MONITORING_KIND '1'";
+
+--error ER_FOREIGN_SERVER_DOESNT_EXIST
+CREATE TABLE t (c INT) ENGINE=Spider COMMENT="WRAPPER 'mysql',SRV 's',MONITORING_KIND '2'";
+CREATE TABLE t (c INT) ENGINE=Spider REMOTE_PORT="1";
+drop table t;
+
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--echo #
+--echo # end of test mdev_32486
+--echo #

storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test → storage/spider/mysql-test/spider/bugfix/t/sysvar_params.test

 --echo
---echo MDEV-31524 Spider variables that double as table params overriding mechanism is buggy
+--echo Testing spider sysvar and table params / options, including default values and overriding mechanisms
 --echo
 
 --disable_query_log
@@ -8,62 +8,115 @@
 --enable_result_log
 --enable_query_log
 
---let $srv=srv_mdev_31524
+# For tests covering overriding mechanism between params and options,
+# including a mix of different levels (partition vs table), see
+# spider/features.engine_defined_attributes.
 SET @old_spider_read_only_mode = @@session.spider_read_only_mode;
-evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
+evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root');
 
-# when the user does not set var nor the table option, the default
+--echo # Cases where table params/options are not set
+# when the user does not set var nor the table param/option, the default
 # value (0 in this case) takes effect.
 set session spider_read_only_mode = default;
 create table t2 (c int);
-eval create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 1 */ insert into t1 values (42);
 drop table t1, t2;
 
-# when the user sets var but not the table option, the var should be
-# take effect.
+# when the user sets var but not the table param/option, the var
+# should be take effect.
 set session spider_read_only_mode = 1;
 create table t2 (c int);
-eval create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
+
 --error 12518
 /* 2 */ insert into t1 values (42);
 drop table t1, t2;
 
-# when the user sets var to -1 and does not set the table option, the
-# default value takes effect.
+# When the user sets a sysvar to -1, it falls back to default
 set session spider_read_only_mode = -1;
 create table t2 (c int);
-eval create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"';
 /* 3 */ insert into t1 values (42);
 drop table t1, t2;
 
+--echo # Cases where table params are set
+# when the user does not set the var, but sets the table param, the
+# table param takes effect
+SET session spider_read_only_mode = default;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "srv",TABLE "t2"';
+--error 12518
+/* 4 */ insert into t1 values (42);
+drop table t1, t2;
+
+# when the user sets both var and table param, the table param takes
+# precedence
+set session spider_read_only_mode = 1;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "0", WRAPPER "mysql", srv "srv",TABLE "t2"';
+/* 5 */ insert into t1 values (42);
+drop table t1, t2;
+
+# when the user sets the var to -1 and sets the table param, the
+# table param takes effect
+SET session spider_read_only_mode = -1;
+create table t2 (c int);
+eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "$srv",TABLE "t2"';
+--error 12518
+/* 6 */ insert into t1 values (42);
+drop table t1, t2;
+
+--echo # Cases where table options are set
 # when the user does not set the var, but sets the table option, the
 # table option should take effect
 SET session spider_read_only_mode = default;
 create table t2 (c int);
-eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "$srv",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider REMOTE_SERVER=srv REMOTE_TABLE=t2 READ_ONLY=1 WRAPPER=mysql;
 --error 12518
-/* 4 */ insert into t1 values (42);
+/* 7 */ insert into t1 values (42);
 drop table t1, t2;
 
 # when the user sets both var and table option, the table option
 # should take precedence
 set session spider_read_only_mode = 1;
 create table t2 (c int);
-eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "0", WRAPPER "mysql", srv "$srv",TABLE "t2"';
-/* 5 */ insert into t1 values (42);
+create table t1 (c int) ENGINE=Spider READ_ONLY=0 REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 8 */ insert into t1 values (42);
 drop table t1, t2;
 
 # when the user sets the var to -1 and sets the table option, the
-# table option should take effect
+# table option takes effect
 SET session spider_read_only_mode = -1;
 create table t2 (c int);
-eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "$srv",TABLE "t2"';
+create table t1 (c int) ENGINE=Spider READ_ONLY=1 REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
 --error 12518
-/* 6 */ insert into t1 values (42);
+/* 9 */ insert into t1 values (42);
+drop table t1, t2;
+
+# the default table option falls back to sysvar
+SET session spider_read_only_mode = 0;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=default REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 10 */ insert into t1 values (42);
 drop table t1, t2;
 
-eval drop server $srv;
+# the default table option falls back to sysvar
+SET session spider_read_only_mode = 1;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=default REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+--error 12518
+/* 11 */ insert into t1 values (42);
+drop table t1, t2;
+
+# the default table option falls back to sysvar
+SET session spider_read_only_mode = -1;
+create table t2 (c int);
+create table t1 (c int) ENGINE=Spider READ_ONLY=default REMOTE_SERVER=srv REMOTE_TABLE=t2 WRAPPER=mysql;
+/* 12 */ insert into t1 values (42);
+drop table t1, t2;
+
+drop server srv;
 
 SET session spider_read_only_mode = @old_spider_read_only_mode;
 --disable_query_log
@@ -71,3 +124,7 @@ SET session spider_read_only_mode = @old_spider_read_only_mode;
 --source ../../t/test_deinit.inc
 --enable_result_log
 --enable_query_log
+
+--echo #
+--echo # End of test sysvar_params
+--echo #

storage/spider/spd_err.h

@@ -70,6 +70,10 @@
 #define ER_SPIDER_INVALID_CONNECT_INFO_START_WITH_NUM_STR "The connect info '%-.64s' for %s cannot start with number"
 #define ER_SPIDER_INVALID_CONNECT_INFO_SAME_NUM 12527
 #define ER_SPIDER_INVALID_CONNECT_INFO_SAME_STR "The connect info '%-.64s' for %s cannot use same name in same table"
+#define ER_SPIDER_INVALID_TABLE_OPTION_NUM 12528
+#define ER_SPIDER_INVALID_TABLE_OPTION_STR "The table option %s=%s is invalid"
+#define ER_SPIDER_COMMENT_CONNECTION_IGNORED_BY_TABLE_OPTIONS_NUM 12529
+#define ER_SPIDER_COMMENT_CONNECTION_IGNORED_BY_TABLE_OPTIONS_STR "The table or partition COMMENT or CONNECTION string '%s' is not used as connection info because spider_ignore_comment is 1 or at least one table option has been specified"
 
 #define ER_SPIDER_CANT_USE_BOTH_INNER_XA_AND_SNAPSHOT_NUM 12601
 #define ER_SPIDER_CANT_USE_BOTH_INNER_XA_AND_SNAPSHOT_STR "Can't use both spider_use_consistent_snapshot = 1 and spider_internal_xa = 1"

storage/spider/spd_param.cc

@@ -2378,6 +2378,30 @@ static MYSQL_THDVAR_BOOL(
 
 SPIDER_THDVAR_VALUE_FUNC(bool, disable_group_by_handler)
 
+static MYSQL_THDVAR_BOOL(
+  suppress_comment_ignored_warning,
+  PLUGIN_VAR_RQCMDARG,
+  "Whether to suppress warnings that table COMMENT or CONNECTION strings "
+  "are ignored due to specified table options",
+  NULL,
+  NULL,
+  FALSE
+);
+
+SPIDER_THDVAR_VALUE_FUNC(bool, suppress_comment_ignored_warning)
+
+static MYSQL_THDVAR_BOOL(
+  ignore_comments,
+  PLUGIN_VAR_RQCMDARG,
+  "Whether to unconditionally ignore COMMENT and CONNECTION strings "
+  "without checking whether table options are specified",
+  NULL,
+  NULL,
+  FALSE
+);
+
+SPIDER_THDVAR_VALUE_FUNC(bool, ignore_comments)
+
 static struct st_mysql_storage_engine spider_storage_engine =
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
@@ -2494,6 +2518,8 @@ static struct st_mysql_sys_var* spider_system_variables[] = {
   MYSQL_SYSVAR(strict_group_by),
   MYSQL_SYSVAR(direct_aggregate),
   MYSQL_SYSVAR(disable_group_by_handler),
+  MYSQL_SYSVAR(suppress_comment_ignored_warning),
+  MYSQL_SYSVAR(ignore_comments),
   NULL
 };
 

storage/spider/spd_param.h

@@ -367,3 +367,5 @@ int spider_param_strict_group_by(
 );
 bool spider_param_direct_aggregate(THD *thd);
 bool spider_param_disable_group_by_handler(THD *thd);
+bool spider_param_suppress_comment_ignored_warning(THD *thd);
+bool spider_param_ignore_comments(THD *thd);