MDEV-22570 Implement wsrep_provider_options as plugin

- Provider options are read from the provider during
  startup, before plugins are initialized.
- New wsrep_provider plugin for which sysvars are generated
  dynamically from options read from the provider.
- The plugin is enabled by option plugin-wsrep-provider=ON.
  If enabled, wsrep_provider_options can no longer be used,
  (an error is raised on attempts to do so).
- Each option is either string, integer, double or bool
- Options can be dynamic / readonly
- Options can be deprecated

Limitations:

- We do not check that the value of a provider option falls
  within a certain range. This type of validation is still
  done in Galera side.
Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/wsrep/r/wsrep_provider_plugin.result

+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+SET GLOBAL wsrep_provider_repl_max_ws_size=1;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+Variable_name	Value
+wsrep_provider_repl_max_ws_size	1
+INSERT INTO t1 VALUES (1);
+ERROR HY000: Got error 5 "Input/output error" during COMMIT
+SET GLOBAL wsrep_provider_repl_max_ws_size=DEFAULT;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+Variable_name	Value
+wsrep_provider_repl_max_ws_size	2147483647
+INSERT INTO t1 VALUES (1);
+SET GLOBAL wsrep_provider_options='repl.max_ws_size=1';
+ERROR HY000: Variable 'wsrep_provider_options' is a read only variable
+INSERT INTO t1 VALUES (2);
+SET GLOBAL wsrep_provider='none';
+ERROR HY000: Variable 'wsrep_provider' is a read only variable
+DROP TABLE t1;
+CALL mtr.add_suppression("transaction size limit");
+CALL mtr.add_suppression("rbr write fail");

mysql-test/suite/wsrep/r/wsrep_provider_plugin_basic.result

+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_socket_recv_buf_size';
+variable_type	global_value
+VARCHAR	auto
+set global wsrep_provider_socket_recv_buf_size = 'foo';
+ERROR 42000: Variable 'socket_recv_buf_size' can't be set to the value of 'foo'
+set global wsrep_provider_socket_recv_buf_size = '1M';
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+Variable_name	Value
+wsrep_provider_socket_recv_buf_size	1M
+set global wsrep_provider_socket_recv_buf_size = default;
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+Variable_name	Value
+wsrep_provider_socket_recv_buf_size	auto
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_evs_send_window';
+variable_type	global_value
+BIGINT	4
+set global wsrep_provider_evs_send_window = -10;
+ERROR 42000: Variable 'evs_send_window' can't be set to the value of '-10'
+set global wsrep_provider_evs_send_window = 10;
+show global variables like 'wsrep_provider_evs_send_window';
+Variable_name	Value
+wsrep_provider_evs_send_window	10
+set global wsrep_provider_evs_send_window = default;
+show global variables like 'wsrep_provider_evs_send_window';
+Variable_name	Value
+wsrep_provider_evs_send_window	4
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_gcs_max_throttle';
+variable_type
+DOUBLE
+set global wsrep_provider_gcs_max_throttle = 1.1;
+ERROR 42000: Variable 'gcs_max_throttle' can't be set to the value of '1.100000'
+set global wsrep_provider_gcs_max_throttle = 0.5;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+Variable_name	Value
+wsrep_provider_gcs_max_throttle	0.500000
+set global wsrep_provider_gcs_max_throttle = default;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+Variable_name	Value
+wsrep_provider_gcs_max_throttle	0.250000
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_cert_log_conflicts';
+variable_type
+BOOLEAN
+set global wsrep_provider_cert_log_conflicts = on;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+Variable_name	Value
+wsrep_provider_cert_log_conflicts	ON
+set global wsrep_provider_cert_log_conflicts = off;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+Variable_name	Value
+wsrep_provider_cert_log_conflicts	OFF
+set global wsrep_provider_cert_log_conflicts = default;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+Variable_name	Value
+wsrep_provider_cert_log_conflicts	OFF
+select read_only from information_schema.system_variables where variable_name = 'wsrep_provider_evs_auto_evict';
+read_only
+YES
+set global wsrep_provider_evs_auto_evict = on;
+ERROR HY000: Variable 'wsrep_provider_evs_auto_evict' is a read only variable
+set global wsrep_provider_gcs_fc_master_slave = default;
+Warnings:
+Warning	1287	'@@wsrep_provider_gcs_fc_master_slave' is deprecated and will be removed in a future release
+call mtr.add_suppression("error setting param");
+call mtr.add_suppression("Unknown parameter");
+call mtr.add_suppression("Setting parameter");

mysql-test/suite/wsrep/t/wsrep_provider_plugin.cnf

+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
+plugin-wsrep-provider=ON

mysql-test/suite/wsrep/t/wsrep_provider_plugin.test

+#
+# Verify that system variables can be modified via wsrep_provider
+# plugin and wsrep_provider/wsrep_provider_options cannot be modified.
+#
+
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+
+SET GLOBAL wsrep_provider_repl_max_ws_size=1;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+
+--error ER_ERROR_DURING_COMMIT
+INSERT INTO t1 VALUES (1);
+
+SET GLOBAL wsrep_provider_repl_max_ws_size=DEFAULT;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+
+INSERT INTO t1 VALUES (1);
+
+# Variable should be read only, must not take effect
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET GLOBAL wsrep_provider_options='repl.max_ws_size=1';
+INSERT INTO t1 VALUES (2);
+
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET GLOBAL wsrep_provider='none';
+
+DROP TABLE t1;
+
+CALL mtr.add_suppression("transaction size limit");
+CALL mtr.add_suppression("rbr write fail");

mysql-test/suite/wsrep/t/wsrep_provider_plugin_basic.cnf

+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
+plugin-wsrep-provider=ON

mysql-test/suite/wsrep/t/wsrep_provider_plugin_basic.test

+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+
+#
+# Test string option
+#
+
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_socket_recv_buf_size';
+--error ER_WRONG_VALUE_FOR_VAR
+set global wsrep_provider_socket_recv_buf_size = 'foo';
+set global wsrep_provider_socket_recv_buf_size = '1M';
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+set global wsrep_provider_socket_recv_buf_size = default;
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+
+
+#
+# Test integer option
+#
+
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_evs_send_window';
+--error ER_WRONG_VALUE_FOR_VAR
+set global wsrep_provider_evs_send_window = -10;
+set global wsrep_provider_evs_send_window = 10;
+show global variables like 'wsrep_provider_evs_send_window';
+set global wsrep_provider_evs_send_window = default;
+show global variables like 'wsrep_provider_evs_send_window';
+
+
+#
+# Test double option
+#
+
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_gcs_max_throttle';
+--error ER_WRONG_VALUE_FOR_VAR
+set global wsrep_provider_gcs_max_throttle = 1.1;
+set global wsrep_provider_gcs_max_throttle = 0.5;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+set global wsrep_provider_gcs_max_throttle = default;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+
+
+#
+# Test bool option
+#
+
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_cert_log_conflicts';
+set global wsrep_provider_cert_log_conflicts = on;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+set global wsrep_provider_cert_log_conflicts = off;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+set global wsrep_provider_cert_log_conflicts = default;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+
+
+#
+# Test read-only option
+#
+
+select read_only from information_schema.system_variables where variable_name = 'wsrep_provider_evs_auto_evict';
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set global wsrep_provider_evs_auto_evict = on;
+
+
+#
+# Test deprecated option (expect warning in result file)
+#
+
+set global wsrep_provider_gcs_fc_master_slave = default;
+
+
+call mtr.add_suppression("error setting param");
+call mtr.add_suppression("Unknown parameter");
+call mtr.add_suppression("Setting parameter");

mysql-test/suite/wsrep/t/wsrep_provider_plugin_defaults.cnf

+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
+plugin-wsrep-provider=ON

mysql-test/suite/wsrep/t/wsrep_provider_plugin_defaults.test

+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_provider%';
+
+--vertical_results
+SELECT * FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES
+WHERE VARIABLE_NAME LIKE 'wsrep_provider_%' AND VARIABLE_NAME NOT IN (
+      'wsrep_provider',
+      'wsrep_provider_options',
+      'wsrep_provider_base_dir',
+      'wsrep_provider_base_port',
+      'wsrep_provider_gcache_dir',
+      'wsrep_provider_gmcast_listen_addr')
+ORDER BY VARIABLE_NAME;

sql/wsrep_mysqld.cc

@@ -56,7 +56,6 @@
 #include <sstream>
 
 /* wsrep-lib */
-Wsrep_server_state* Wsrep_server_state::m_instance;
 
 my_bool wsrep_emulate_bin_log= FALSE; // activating parts of binlog interface
 my_bool wsrep_preordered_opt= FALSE;
@@ -873,7 +872,8 @@ int wsrep_init()
   {
     // enable normal operation in case no provider is specified
     global_system_variables.wsrep_on= 0;
-    int err= Wsrep_server_state::instance().load_provider(wsrep_provider, wsrep_provider_options ? wsrep_provider_options : "");
+    int err= Wsrep_server_state::init_provider(
+        wsrep_provider, wsrep_provider_options ? wsrep_provider_options : "");
     if (err)
     {
       DBUG_PRINT("wsrep",("wsrep::init() failed: %d", err));
@@ -915,7 +915,7 @@ int wsrep_init()
                 "wsrep_trx_fragment_size to 0 or use wsrep_provider that "
                 "supports streaming replication.",
                 wsrep_provider, global_system_variables.wsrep_trx_fragment_size);
-    Wsrep_server_state::instance().unload_provider();
+    Wsrep_server_state::instance().deinit_provider();
     Wsrep_server_state::deinit_provider_services();
     return 1;
   }
@@ -931,7 +931,6 @@ int wsrep_init()
 
   WSREP_DEBUG("SR storage init for: %s",
               (wsrep_SR_store_type == WSREP_SR_STORE_TABLE) ? "table" : "void");
-
   return 0;
 }
 
@@ -1006,6 +1005,11 @@ void wsrep_init_startup (bool sst_first)
   wsrep_create_rollbacker();
   wsrep_create_appliers(1);
 
+  if (Wsrep_server_state::init_options())
+  {
+    WSREP_WARN("Failed to initialize provider options");
+  }
+
   Wsrep_server_state& server_state= Wsrep_server_state::instance();
   /*
     If the SST happens before server initialization, wait until the server
@@ -1031,7 +1035,7 @@ void wsrep_deinit(bool free_options)
   DBUG_ASSERT(wsrep_inited == 1);
   WSREP_DEBUG("wsrep_deinit");
 
-  Wsrep_server_state::instance().unload_provider();
+  Wsrep_server_state::deinit_provider();
   Wsrep_server_state::deinit_provider_services();
 
   provider_name[0]=    '\0';

sql/wsrep_server_state.cc

-/* Copyright 2018 Codership Oy <info@codership.com>
+/* Copyright 2018-2022 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,6 +19,7 @@
 #include "wsrep_allowlist_service.h"
 #include "wsrep_event_service.h"
 #include "wsrep_binlog.h" /* init/deinit group commit */
+#include "wsrep_plugin.h" /* make/destroy sysvar helpers */
 
 mysql_mutex_t LOCK_wsrep_server_state;
 mysql_cond_t  COND_wsrep_server_state;
@@ -30,6 +31,10 @@ PSI_cond_key  key_COND_wsrep_server_state;
 
 wsrep::provider::services Wsrep_server_state::m_provider_services;
 
+Wsrep_server_state* Wsrep_server_state::m_instance;
+std::unique_ptr<wsrep::provider_options> Wsrep_server_state::m_options;
+std::vector<st_mysql_sys_var*> Wsrep_server_state::m_sysvars;
+
 Wsrep_server_state::Wsrep_server_state(const std::string& name,
                                        const std::string& incoming_address,
                                        const std::string& address,
@@ -76,6 +81,48 @@ void Wsrep_server_state::init_once(const std::string& name,
   }
 }
 
+int Wsrep_server_state::init_provider(const std::string& provider,
+                                      const std::string& options)
+{
+  DBUG_ASSERT(m_instance);
+  int ret= m_instance->load_provider(provider, options);
+  if (ret)
+  {
+    WSREP_ERROR("Failed to load provider %s with options %s",
+                provider.c_str(), options.c_str());
+    return ret;
+  }
+  return 0;
+}
+
+int Wsrep_server_state::init_options()
+{
+  if (!m_instance) return 1;
+  m_options= std::unique_ptr<wsrep::provider_options>(
+     new wsrep::provider_options(m_instance->provider()));
+  int ret= m_options->initial_options();
+  if (ret)
+  {
+    WSREP_ERROR("Failed to initialize provider options");
+    m_options = nullptr;
+    m_instance->unload_provider();
+    return ret;
+  }
+  m_options->for_each([](wsrep::provider_options::option *opt) {
+    struct st_mysql_sys_var *var= wsrep_make_sysvar_for_option(opt);
+    m_sysvars.push_back(var);
+  });
+  m_sysvars.push_back(nullptr);
+  wsrep_provider_plugin_set_sysvars(&m_sysvars[0]);
+  return 0;
+}
+
+void Wsrep_server_state::deinit_provider()
+{
+  m_options = nullptr;
+  m_instance->unload_provider();
+}
+
 void Wsrep_server_state::destroy()
 {
   if (m_instance)
@@ -84,6 +131,14 @@ void Wsrep_server_state::destroy()
     m_instance= 0;
     mysql_mutex_destroy(&LOCK_wsrep_server_state);
     mysql_cond_destroy(&COND_wsrep_server_state);
+    for (auto var : m_sysvars)
+    {
+      if (var)
+      {
+        wsrep_destroy_sysvar(var);
+      }
+    }
+    m_sysvars.clear();
   }
 }
 

sql/wsrep_server_state.h

-/* Copyright 2018 Codership Oy <info@codership.com>
+/* Copyright 2018-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,6 +19,7 @@
 /* wsrep-lib */
 #include "wsrep/server_state.hpp"
 #include "wsrep/provider.hpp"
+#include "wsrep/provider_options.hpp"
 
 /* implementation */
 #include "wsrep_server_service.h"
@@ -34,6 +35,10 @@ class Wsrep_server_state : public wsrep::server_state
                         const std::string& working_dir,
                         const wsrep::gtid& initial_position,
                         int max_protocol_version);
+  static int init_provider(const std::string& provider,
+                           const std::string& options);
+  static int init_options();
+  static void deinit_provider();
   static void destroy();
 
   static Wsrep_server_state& instance()
@@ -51,6 +56,11 @@ class Wsrep_server_state : public wsrep::server_state
     return instance().provider();
   }
 
+  static wsrep::provider_options* get_options()
+  {
+    return m_options.get();
+  }
+
   static bool has_capability(int capability)
   {
     return (get_provider().capabilities() & capability);
@@ -77,7 +87,11 @@ class Wsrep_server_state : public wsrep::server_state
   Wsrep_server_service m_service;
   static wsrep::provider::services m_provider_services;
   static Wsrep_server_state* m_instance;
-
+  static std::unique_ptr<wsrep::provider_options> m_options;
+  // Sysvars for provider plugin. We keep these here because
+  // they are allocated dynamically and must be freed at some
+  // point during shutdown (after the plugin is deinitialized).
+  static std::vector<st_mysql_sys_var *> m_sysvars;
 };
 
 #endif // WSREP_SERVER_STATE_H

sql/wsrep_var.cc

@@ -27,6 +27,7 @@
 #include <cstdlib>
 #include "wsrep_trans_observer.h"
 #include "wsrep_server_state.h"
+#include "wsrep_plugin.h" /* wsrep_provider_plugin_is_enabled() */
 
 ulong   wsrep_reject_queries;
 
@@ -93,6 +94,11 @@ static bool refresh_provider_options()
   }
 }
 
+bool wsrep_refresh_provider_options()
+{
+  return refresh_provider_options();
+}
+
 void wsrep_set_wsrep_on(THD* thd)
 {
   if (thd)
@@ -447,6 +453,12 @@ static int wsrep_provider_verify (const char* provider_str)
 
 bool wsrep_provider_check (sys_var *self, THD* thd, set_var* var)
 {
+  if (wsrep_provider_plugin_enabled())
+  {
+    my_error(ER_INCORRECT_GLOBAL_LOCAL_VAR, MYF(0), var->var->name.str, "read only");
+    return true;
+  }
+
   char wsrep_provider_buf[FN_REFLEN];
 
   if ((! var->save_result.string_value.str) ||
@@ -538,6 +550,11 @@ bool wsrep_provider_options_check(sys_var *self, THD* thd, set_var* var)
     my_message(ER_WRONG_ARGUMENTS, "WSREP (galera) not started", MYF(0));
     return true;
   }
+  if (wsrep_provider_plugin_enabled())
+  {
+    my_error(ER_INCORRECT_GLOBAL_LOCAL_VAR, MYF(0), var->var->name.str, "read only");
+    return true;
+  }
   return false;
 }
 

sql/wsrep_var.h

-/* Copyright (C) 2013-2021 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2022 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -37,6 +37,7 @@ class THD;
 
 int wsrep_init_vars();
 void wsrep_set_wsrep_on(THD *thd);
+bool wsrep_refresh_provider_options();
 
 #define CHECK_ARGS   (sys_var *self, THD* thd, set_var *var)
 #define UPDATE_ARGS  (sys_var *self, THD* thd, enum_var_type type)