MDEV-34867 engine S3 cause 500 error for huawei buckets

Add support for removing the Content-Type header to the S3 engine. This
is required for compatibility with some S3 providers.

This also adds a provider option to the S3 engine which will turn on
relevant compatibility options for specific providers.

This was required for getting MariaDB S3 engine to work with "Huawei
Cloud S3".
To get Huawei S3 storage to work on has set one of the following
S3 options:
s3_provider=Huawei
s3_ssl_no_verify=1

Author: Andrew Hutchings <andrew@mariadb.org>

mysql-test/suite/s3/basic.result

@@ -130,16 +130,19 @@ s3_block_size	X
 s3_bucket	X
 s3_debug	X
 s3_host_name	X
+s3_no_content_type	X
 s3_pagecache_age_threshold	X
 s3_pagecache_buffer_size	X
 s3_pagecache_division_limit	X
 s3_pagecache_file_hash_size	X
 s3_port	X
 s3_protocol_version	X
+s3_provider	X
 s3_region	X
 s3_replicate_alter_as_create_select	X
 s3_secret_key	X
 s3_slave_ignore_updates	X
+s3_ssl_no_verify	X
 s3_use_http	X
 show variables like "s3_slave%";
 Variable_name	Value

mysql-test/suite/s3/my.cnf

@@ -12,6 +12,8 @@ s3-secret-key=@ENV.S3_SECRET_KEY
 s3-region=@ENV.S3_REGION
 s3-port=@ENV.S3_PORT
 s3-use-http=@ENV.S3_USE_HTTP
+s3-ssl-no-verify=@ENV.S3_SSL_NO_VERIFY
+s3-provider=@ENV.S3_PROVIDER
 
 #s3-host-name=s3.amazonaws.com
 #s3-protocol-version=Amazon

mysql-test/suite/s3/slave.cnf

@@ -10,6 +10,8 @@ s3-secret-key=@ENV.S3_SECRET_KEY
 s3-region=@ENV.S3_REGION
 s3-port=@ENV.S3_PORT
 s3-use-http=@ENV.S3_USE_HTTP
+s3-ssl-no-verify=@ENV.S3_SSL_NO_VERIFY
+s3-provider=@ENV.S3_PROVIDER
 
 # You can change the following when running the tests against
 # your own S3 setup

mysql-test/suite/s3/suite.pm

@@ -20,6 +20,8 @@ if(connect(SOCK, $paddr))
   $ENV{'S3_REGION'} = "";
   $ENV{'S3_PROTOCOL_VERSION'} = "Auto";
   $ENV{'S3_USE_HTTP'} = "ON";
+  $ENV{'S3_SSL_NO_VERIFY'} = "OFF";
+  $ENV{'S3_PROVIDER'} = "Default";
 }
 else
 {
@@ -62,6 +64,17 @@ else
   {
     $ENV{'S3_USE_HTTP'} = "OFF";
   }
+
+  if (!$ENV{'S3_SSL_NO_VERIFY'})
+  {
+    $ENV{'S3_SSL_NO_VERIFY'} = "OFF";
+  }
+
+  if (!$ENV{'S3_PROVIDER'})
+  {
+    $ENV{'S3_PROVIDER'} = "Default";
+  }
+
 }
 bless { };
 

storage/maria/aria_s3_copy.cc

@@ -41,7 +41,10 @@ static const char *opt_database;
 static const char *opt_s3_bucket="MariaDB";
 static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug;
 static my_bool opt_s3_use_http;
+static my_bool opt_s3_ssl_no_verify;
+static my_bool opt_s3_no_content_type;
 static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1;
+static ulong opt_provider= 0;
 static ulong opt_block_size;
 static ulong opt_s3_port;
 static char **default_argv=0;
@@ -73,6 +76,13 @@ static struct my_option my_long_options[] =
   {"s3_use_http", 'P', "If true, force use of HTTP protocol",
    (char**) &opt_s3_use_http, (char**) &opt_s3_use_http,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"s3_ssl_no_verify", 's', "If true, verification of the S3 endpoint SSL "
+   "certificate is disabled",
+   (char**) &opt_s3_ssl_no_verify, (char**) &opt_s3_ssl_no_verify,
+   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"s3_no_content_type", 'n', "If true, disables the Content-Type header",
+   (char**) &opt_s3_no_content_type, (char**) &opt_s3_no_content_type,
+   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"compress", 'c', "Use compression", &opt_compression, &opt_compression,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"op", 'o', "Operation to execute. One of 'from_s3', 'to_s3' or "
@@ -92,6 +102,10 @@ static struct my_option my_long_options[] =
    "Note: \"Legacy\", \"Original\" and \"Amazon\" are deprecated.",
    &opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib,
    GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"s3_provider", 'R', "Enable S3 provider specific compatibility tweaks "
+   "\"Default\", \"Amazon\", or \"Huawei\".",
+   &opt_provider, &opt_provider, &s3_provider_typelib,
+   GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"force", 'f', "Force copy even if target exists",
    &opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose,
@@ -220,6 +234,19 @@ int main(int argc, char** argv)
 
   ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size);
 
+  /* Provider specific overrides */
+  switch (opt_provider)
+  {
+    case 0: /* Default */
+      break;
+    case 1: /* Amazon */
+      opt_protocol_version = 5;
+      break;
+    case 2: /* Huawei */
+      opt_s3_no_content_type = 1;
+      break;
+  }
+
   if (opt_protocol_version > 2)
   {
     uint8_t protocol_version;
@@ -245,6 +272,11 @@ int main(int argc, char** argv)
   if (opt_s3_use_http)
     ms3_set_option(global_s3_client, MS3_OPT_USE_HTTP, NULL);
 
+  if (opt_s3_ssl_no_verify)
+    ms3_set_option(global_s3_client, MS3_OPT_DISABLE_SSL_VERIFY, NULL);
+
+  if (opt_s3_no_content_type)
+    ms3_set_option(global_s3_client, MS3_OPT_NO_CONTENT_TYPE, NULL);
 
   for (; *argv ; argv++)
   {

storage/maria/ha_s3.cc

@@ -78,7 +78,7 @@
 #define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
 
 static PAGECACHE s3_pagecache;
-static ulong s3_block_size, s3_protocol_version;
+static ulong s3_block_size, s3_protocol_version, s3_provider;
 static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold;
 static ulong s3_pagecache_file_hash_size;
 static ulonglong s3_pagecache_buffer_size;
@@ -86,6 +86,8 @@ static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region;
 static char *s3_host_name;
 static int s3_port;
 static my_bool s3_use_http;
+static my_bool s3_ssl_no_verify;
+static my_bool s3_no_content_type;
 static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0;
 static my_bool s3_debug= 0, s3_slave_ignore_updates= 0;
 static my_bool s3_replicate_alter_as_create_select= 0;
@@ -222,6 +224,10 @@ static MYSQL_SYSVAR_BOOL(use_http, s3_use_http,
        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
       "If true, force use of HTTP protocol",
        NULL /*check*/, NULL /*update*/, 0 /*default*/);
+static MYSQL_SYSVAR_BOOL(ssl_no_verify, s3_ssl_no_verify,
+       PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+      "If true, SSL certificate verifiction for the S3 endpoint is disabled",
+       NULL, NULL, 0);
 static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key,
        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC,
       "AWS access key",
@@ -234,6 +240,15 @@ static MYSQL_SYSVAR_STR(region, s3_region,
        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
       "AWS region",
        0, 0, "");
+static MYSQL_SYSVAR_BOOL(no_content_type, s3_no_content_type,
+       PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+      "If true, disables the Content-Type header, required for some providers",
+      NULL, NULL, 0);
+static MYSQL_SYSVAR_ENUM(provider, s3_provider,
+                         PLUGIN_VAR_RQCMDARG,
+                         "Enable S3 provider specific compatibility tweaks "
+                         "\"Default\", \"Amazon\", or \"Huawei\". ",
+                         NULL, NULL, 0, &s3_provider_typelib);
 
 ha_create_table_option s3_table_option_list[]=
 {
@@ -319,6 +334,9 @@ static my_bool s3_info_init(S3_INFO *info)
   lex_string_set(&info->host_name,  s3_host_name);
   info->port= s3_port;
   info->use_http= s3_use_http;
+  info->ssl_no_verify= s3_ssl_no_verify;
+  info->no_content_type = s3_no_content_type;
+  info->provider= s3_provider;
   lex_string_set(&info->access_key, s3_access_key);
   lex_string_set(&info->secret_key, s3_secret_key);
   lex_string_set(&info->region,     s3_region);
@@ -1120,12 +1138,15 @@ static struct st_mysql_sys_var* system_variables[]= {
   MYSQL_SYSVAR(host_name),
   MYSQL_SYSVAR(port),
   MYSQL_SYSVAR(use_http),
+  MYSQL_SYSVAR(ssl_no_verify),
   MYSQL_SYSVAR(bucket),
   MYSQL_SYSVAR(access_key),
   MYSQL_SYSVAR(secret_key),
   MYSQL_SYSVAR(region),
   MYSQL_SYSVAR(slave_ignore_updates),
   MYSQL_SYSVAR(replicate_alter_as_create_select),
+  MYSQL_SYSVAR(no_content_type),
+  MYSQL_SYSVAR(provider),
   NULL
 };
 

libmarias3 @ 5e6aa32f

-Subproject commit a81724ab07bd28e16bf431419c24b6362d5894fc
+Subproject commit 5e6aa32f96ebdbcaf32b90d6182685156f8198b5

storage/maria/s3_func.c

@@ -43,6 +43,9 @@ static const char *protocol_types[]= {"Auto", "Original", "Amazon", "Legacy", "P
 TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"",
                               protocol_types, NULL};
 
+static const char *providers[]= {"Default", "Amazon", "Huawei", NullS};
+TYPELIB s3_provider_typelib = {array_elements(providers)-1,"",providers, NULL};
+
 /******************************************************************************
  Allocations handler for libmarias3
  To be removed when we do the init allocation in mysqld.cc
@@ -154,6 +157,20 @@ ms3_st *s3_open_connection(S3_INFO *s3)
                     errno, ms3_error(errno));
     my_errno= HA_ERR_NO_SUCH_TABLE;
   }
+
+  /* Provider specific overrides */
+  switch (s3->provider)
+  {
+    case 0: /* Default */
+      break;
+    case 1: /* Amazon */
+      s3->protocol_version = 5;
+      break;
+    case 2: /* Huawei */
+      s3->no_content_type = 1;
+      break;
+  }
+
   if (s3->protocol_version > 2)
   {
     uint8_t protocol_version;
@@ -177,6 +194,12 @@ ms3_st *s3_open_connection(S3_INFO *s3)
   if (s3->use_http)
     ms3_set_option(s3_client, MS3_OPT_USE_HTTP, NULL);
 
+  if (s3->ssl_no_verify)
+    ms3_set_option(s3_client, MS3_OPT_DISABLE_SSL_VERIFY, NULL);
+
+  if (s3->no_content_type)
+    ms3_set_option(s3_client, MS3_OPT_NO_CONTENT_TYPE, NULL);
+
   return s3_client;
 }
 

storage/maria/s3_func.h

@@ -38,6 +38,7 @@ extern struct s3_func {
 } s3f;
 
 extern TYPELIB s3_protocol_typelib;
+extern TYPELIB s3_provider_typelib;
 
 /* Store information about a s3 connection */
 
@@ -47,6 +48,8 @@ struct s3_info
   LEX_CSTRING access_key, secret_key, region, bucket, host_name;
   int port; // 0 means 'Use default'
   my_bool use_http;
+  my_bool ssl_no_verify;
+  my_bool no_content_type;
 
   /* Will be set by caller or by ma_open() */
   LEX_CSTRING database, table;
@@ -63,6 +66,8 @@ struct s3_info
 
   /* Protocol for the list bucket API call. 1 for Amazon, 2 for some others */
   uint8_t protocol_version;
+
+  uint8_t provider;
 };