revision-id: cd0e4f28fd45433e98f28933bc3ed103b34b64ac (mariadb-10.5.4-167-gcd0e4f28fd4) parent(s): 0af6e525212f81c2da8185817c6ca2e7c3ac036a author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-04 00:15:22 +0300 message: MDEV-23662: Make S3 Storage Engine usable with MinIO Add @@s3_port and @@s3_use_http. One can set these to access the local MinIO, for example. --- mysql-test/suite/s3/basic.result | 2 ++ mysql-test/suite/s3/my.cnf | 12 ++++++++++++ mysql-test/suite/s3/slave.cnf | 12 ++++++++++++ storage/maria/ha_s3.cc | 15 +++++++++++++++ storage/maria/s3_func.c | 6 ++++++ storage/maria/s3_func.h | 2 ++ 6 files changed, 49 insertions(+) diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index 9cddca79fc5..cf7d7377d92 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -94,11 +94,13 @@ s3_pagecache_age_threshold X s3_pagecache_buffer_size X s3_pagecache_division_limit X s3_pagecache_file_hash_size X +s3_port X s3_protocol_version X s3_region X s3_replicate_alter_as_create_select X s3_secret_key X s3_slave_ignore_updates X +s3_use_http X show variables like "s3_slave%"; Variable_name Value s3_slave_ignore_updates OFF diff --git a/mysql-test/suite/s3/my.cnf b/mysql-test/suite/s3/my.cnf index 6f8c27d4d0d..e810449d5b3 100644 --- a/mysql-test/suite/s3/my.cnf +++ b/mysql-test/suite/s3/my.cnf @@ -10,3 +10,15 @@ s3=ON #s3-access-key=... #s3-secret-key=... #s3-region=eu-north-1 + +## +## Configuration for local MinIO +## +s3-host-name="127.0.0.1" +# Note: s3-host-name="localhost" doesn't work. It causes +# libmarias3 to use the wrong variant of the protocol. +s3-bucket=storage-engine +s3-access-key=minioadmin +s3-secret-key=minioadmin +s3-port=9000 +s3-use-http=ON diff --git a/mysql-test/suite/s3/slave.cnf b/mysql-test/suite/s3/slave.cnf index 250a46e6322..c5e7f466226 100644 --- a/mysql-test/suite/s3/slave.cnf +++ b/mysql-test/suite/s3/slave.cnf @@ -6,3 +6,15 @@ s3=ON #s3-access-key=... #s3-secret-key=... #s3-region=eu-north-1 + +## +## Configuration for local MinIO +## +s3-host-name="127.0.0.1" +# Note: s3-host-name="localhost" doesn't work. It causes +# libmarias3 to use the wrong variant of the protocol. +s3-bucket=storage-engine +s3-access-key=minioadmin +s3-secret-key=minioadmin +s3-port=9000 +s3-use-http=ON diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index 1aae5a5b81f..190d99035a5 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -80,6 +80,8 @@ static ulong s3_pagecache_file_hash_size; static ulonglong s3_pagecache_buffer_size; static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; static char *s3_host_name; +static int s3_port; +static my_bool s3_use_http; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; static my_bool s3_debug= 0, s3_slave_ignore_updates= 0; static my_bool s3_replicate_alter_as_create_select= 0; @@ -181,6 +183,15 @@ static MYSQL_SYSVAR_STR(host_name, s3_host_name, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "AWS host name", 0, 0, DEFAULT_AWS_HOST_NAME); +static MYSQL_SYSVAR_INT(port, s3_port, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Port number to connect to (0 means use default)", + NULL /*check*/, NULL /*update*/, 0 /*default*/, + 0 /*min*/, 65535 /*max*/, 1 /*blk*/); +static MYSQL_SYSVAR_BOOL(use_http, s3_use_http, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "If true, force use of HTTP protocol", + NULL /*check*/, NULL /*update*/, 0 /*default*/); static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, "AWS access key", @@ -276,6 +287,8 @@ static my_bool s3_info_init(S3_INFO *info) return 1; info->protocol_version= (uint8_t) s3_protocol_version; lex_string_set(&info->host_name, s3_host_name); + info->port= s3_port; + info->use_http= s3_use_http; lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->region, s3_region); @@ -1050,6 +1063,8 @@ static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(pagecache_file_hash_size), MYSQL_SYSVAR(host_name), + MYSQL_SYSVAR(port), + MYSQL_SYSVAR(use_http), MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(secret_key), diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index bbaf048d534..9f40790a371 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -157,6 +157,12 @@ ms3_st *s3_open_connection(S3_INFO *s3) if (s3->protocol_version) ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, &s3->protocol_version); + if (s3->port) + ms3_set_option(s3_client, MS3_OPT_PORT_NUMBER, &s3->port); + + if (s3->use_http) + ms3_set_option(s3_client, MS3_OPT_USE_HTTP, NULL); + return s3_client; } diff --git a/storage/maria/s3_func.h b/storage/maria/s3_func.h index 8981517dd4d..bd5275494bc 100644 --- a/storage/maria/s3_func.h +++ b/storage/maria/s3_func.h @@ -45,6 +45,8 @@ typedef struct s3_info { /* Connection strings */ LEX_CSTRING access_key, secret_key, region, bucket, host_name; + int port; // 0 means 'Use default' + my_bool use_http; /* Will be set by caller or by ma_open() */ LEX_CSTRING database, table;