- commits - lists.mariadb.org

[Commits] 5d9386892b5: MDEV-4633: multi_source.simple test fails sporadically
by sujatha 08 Sep '20

08 Sep '20

revision-id: 5d9386892b58477c4e28de3e40d930b44f5b42d4 (mariadb-10.1.43-274-g5d9386892b5) parent(s): 3ee2422624ffb3d7ffefff8db7ef9398816299bc author: Sujatha committer: Sujatha timestamp: 2020-09-08 13:10:36 +0530 message: MDEV-4633: multi_source.simple test fails sporadically Analysis: ======== Writes to 'rli->log_space_total' needs to be synchronized, otherwise both SQL_THREAD and IO_THREAD can try to modify the variable simultaneously resulting in incorrect rli->log_space_total. In the current test scenario SQL_THREAD is trying to decrement 'rli->log_space_total' in 'purge_first_log' and IO_THREAD is trying to increment the 'rli->log_space_total' in 'queue_event' simultaneously. Hence test occasionally fails with result mismatch. Fix: === Convert 'rli->log_space_total' variable to atomic type. --- sql/log.cc | 4 +++- sql/log.h | 4 +++- sql/rpl_rli.cc | 14 ++++++++------ sql/slave.cc | 19 +++++++++++++++---- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index 8049b94bab1..90d5c46a15f 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -4400,7 +4400,9 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) 0, 0, &log_space_reclaimed); mysql_mutex_lock(&rli->log_space_lock); - rli->log_space_total-= log_space_reclaimed; + my_atomic_add64_explicit((volatile int64*)(&rli->log_space_total), + (-(ulonglong)log_space_reclaimed), + MY_MEMORY_ORDER_RELAXED); mysql_cond_broadcast(&rli->log_space_cond); mysql_mutex_unlock(&rli->log_space_lock); diff --git a/sql/log.h b/sql/log.h index 277e5c6f69c..9e3807e9939 100644 --- a/sql/log.h +++ b/sql/log.h @@ -712,7 +712,9 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG char buf1[22],buf2[22]; #endif DBUG_ENTER("harvest_bytes_written"); - (*counter)+=bytes_written; + + my_atomic_add64_explicit((volatile int64*)(counter), bytes_written, + MY_MEMORY_ORDER_RELAXED); DBUG_PRINT("info",("counter: %s bytes_written: %s", llstr(*counter,buf1), llstr(bytes_written,buf2))); bytes_written=0; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index c196a65809a..8815af4ea4a 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -140,8 +140,8 @@ int init_relay_log_info(Relay_log_info* rli, rli->slave_skip_counter=0; rli->abort_pos_wait=0; rli->log_space_limit= relay_log_space_limit; - rli->log_space_total= 0; - + my_atomic_store64_explicit((volatile int64*)(&rli->log_space_total), 0, + MY_MEMORY_ORDER_RELAXED); char pattern[FN_REFLEN]; (void) my_realpath(pattern, slave_load_tmpdir, 0); if (fn_format(pattern, PREFIX_SQL_LOAD, pattern, "", @@ -467,7 +467,8 @@ static inline int add_relay_log(Relay_log_info* rli,LOG_INFO* linfo) linfo->log_file_name); DBUG_RETURN(1); } - rli->log_space_total += s.st_size; + my_atomic_add64_explicit((volatile int64*)(&rli->log_space_total), + s.st_size, MY_MEMORY_ORDER_RELAXED); DBUG_PRINT("info",("log_space_total: %llu", rli->log_space_total)); DBUG_RETURN(0); } @@ -477,7 +478,8 @@ static int count_relay_log_space(Relay_log_info* rli) { LOG_INFO linfo; DBUG_ENTER("count_relay_log_space"); - rli->log_space_total= 0; + my_atomic_store64_explicit((volatile int64*)(&rli->log_space_total), 0, + MY_MEMORY_ORDER_RELAXED); if (rli->relay_log.find_log_pos(&linfo, NullS, 1)) { sql_print_error("Could not find first log while counting relay log space"); @@ -1202,8 +1204,8 @@ int purge_relay_logs(Relay_log_info* rli, THD *thd, bool just_reset, strmake_buf(rli->group_relay_log_name, rli->relay_log.get_log_fname()); strmake_buf(rli->event_relay_log_name, rli->relay_log.get_log_fname()); rli->group_relay_log_pos= rli->event_relay_log_pos= BIN_LOG_HEADER_SIZE; - rli->log_space_total= 0; - + my_atomic_store64_explicit((volatile int64*)(&rli->log_space_total), 0, + MY_MEMORY_ORDER_RELAXED); if (count_relay_log_space(rli)) { *errmsg= "Error counting relay log space"; diff --git a/sql/slave.cc b/sql/slave.cc index 06f2b0d955a..9f4b0f53fb4 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2371,7 +2371,10 @@ static bool wait_for_relay_log_space(Relay_log_info* rli) &rli->log_space_lock, &stage_waiting_for_relay_log_space, &old_stage); - while (rli->log_space_limit < rli->log_space_total && + while (rli->log_space_limit < + (ulonglong)my_atomic_load64_explicit((volatile int64*) + (&rli->log_space_total), + MY_MEMORY_ORDER_RELAXED) && !(slave_killed=io_slave_killed(mi)) && !rli->ignore_log_space_limit) mysql_cond_wait(&rli->log_space_cond, &rli->log_space_lock); @@ -2912,7 +2915,10 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, protocol->store(mi->rli.last_error().message, &my_charset_bin); protocol->store((uint32) mi->rli.slave_skip_counter); protocol->store((ulonglong) mi->rli.group_master_log_pos); - protocol->store((ulonglong) mi->rli.log_space_total); + protocol->store((ulonglong) + my_atomic_load64_explicit((volatile int64*) + (&mi->rli.log_space_total), + MY_MEMORY_ORDER_RELAXED)); protocol->store( mi->rli.until_condition==Relay_log_info::UNTIL_NONE ? "None": @@ -4428,7 +4434,9 @@ Stopping slave I/O thread due to out-of-memory error from master"); #endif if (rli->log_space_limit && rli->log_space_limit < - rli->log_space_total && + (ulonglong) my_atomic_load64_explicit((volatile int64*) + (&rli->log_space_total), + MY_MEMORY_ORDER_RELAXED) && !rli->ignore_log_space_limit) if (wait_for_relay_log_space(rli)) { @@ -7031,7 +7039,10 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) is are able to rotate and purge sometime soon. */ if (rli->log_space_limit && - rli->log_space_limit < rli->log_space_total) + rli->log_space_limit < + (ulonglong) my_atomic_load64_explicit((volatile int64*) + (&rli->log_space_total), + MY_MEMORY_ORDER_RELAXED)) { /* force rotation if not in an unfinished group */ rli->sql_force_rotate_relay= !rli->is_in_group();

1 0

[Commits] 683cb88de49: Update libmarias3 to PR#92: updated PR#91, adds capability to support port number
by Sergei Petrunia 07 Sep '20

07 Sep '20

revision-id: 683cb88de49b53073fa5cd36150b233df3e04352 (mariadb-10.5.4-171-g683cb88de49) parent(s): 5d12b119d7506b6b3e6f85a3d4b7dacbb98df2b5 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-07 16:07:11 +0300 message: Update libmarias3 to PR#92: updated PR#91, adds capability to support port number --- storage/maria/libmarias3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/maria/libmarias3 b/storage/maria/libmarias3 index 919024c98b9..7aa450e3a02 160000 --- a/storage/maria/libmarias3 +++ b/storage/maria/libmarias3 @@ -1 +1 @@ -Subproject commit 919024c98b965c978e872da7df0c31db7515035b +Subproject commit 7aa450e3a02a20ef62616666a2b582f4d07fb5b3

1 0

[Commits] 5d12b119d75: Update libmarias3 to PR#91 which adds capability to specify port number
by Sergei Petrunia 05 Sep '20

05 Sep '20

revision-id: 5d12b119d7506b6b3e6f85a3d4b7dacbb98df2b5 (mariadb-10.5.4-170-g5d12b119d75) parent(s): 5020dc0bdc38c7706995f20928160065ff8b40b2 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-05 22:24:32 +0300 message: Update libmarias3 to PR#91 which adds capability to specify port number --- storage/maria/libmarias3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/maria/libmarias3 b/storage/maria/libmarias3 index d172e86c162..919024c98b9 160000 --- a/storage/maria/libmarias3 +++ b/storage/maria/libmarias3 @@ -1 +1 @@ -Subproject commit d172e86c16224b4e0229ca6f102e662a2315aeff +Subproject commit 919024c98b965c978e872da7df0c31db7515035b

1 0

[Commits] 01f8db9: Followup to PR#91: use curl_easy_setopt() instead of putting port# into the URI.
by Sergei Petrunia 05 Sep '20

05 Sep '20

revision-id: 01f8db9ccde4318ca924c44caebc941a290b1713 (libmarias3-3.1.2-5-g01f8db9) parent(s): 919024c98b965c978e872da7df0c31db7515035b author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-05 20:12:28 +0300 message: Followup to PR#91: use curl_easy_setopt() instead of putting port# into the URI. --- src/request.c | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/request.c b/src/request.c index cd819c2..80a32d0 100644 --- a/src/request.c +++ b/src/request.c @@ -51,7 +51,7 @@ static void set_error_nocopy(ms3_st *ms3, char *error) ms3->last_error = error; } -static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, +static uint8_t build_request_uri(CURL *curl, const char *base_domain, const char *bucket, const char *object, const char *query, bool use_http, uint8_t protocol_version) { @@ -61,14 +61,6 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, const char *http_protocol = "http"; const char *https_protocol = "https"; const char *protocol; - char port_str[32]; // ":portnumber" or empty string - - if (port) - { - snprintf(port_str, sizeof(port_str) - 1, ":%d", port); - } - else - port_str[0]= 0; // empty string if (base_domain) { @@ -91,43 +83,43 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, if (query) { if (path_parts + strlen(domain) + strlen(bucket) + strlen(object) + strlen( - query) + strlen(port_str) >= MAX_URI_LENGTH - 1) + query) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s?%s", protocol, - domain, port_str, bucket, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s?%s", protocol, + domain, bucket, object, query); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s?%s", protocol, - bucket, domain, port_str, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s?%s", protocol, + bucket, domain, object, query); } } else { if (path_parts + strlen(domain) + strlen(bucket) + strlen( - object) + strlen(port_str) >= MAX_URI_LENGTH - 1) + object) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s", protocol, - domain, port_str, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s", protocol, + domain, bucket, object); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s", protocol, - bucket, domain, port_str, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s", protocol, + bucket, domain, object); } } @@ -725,8 +717,8 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, ms3->query_buffer); } - res = build_request_uri(curl, ms3->base_domain, ms3->port, bucket, path, - query, ms3->use_http, ms3->protocol_version); + res = build_request_uri(curl, ms3->base_domain, bucket, path, query, + ms3->use_http, ms3->protocol_version); if (res) { @@ -783,6 +775,9 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); } + if (ms3->port) + curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem);

1 0

[Commits] 29a6121: Add an option to specify port number
by Sergei Petrunia 05 Sep '20

05 Sep '20

revision-id: 29a6121396da9fc73e3deee7695f7cb95f7121fe (libmarias3-3.1.2-3-g29a6121) parent(s): d172e86c16224b4e0229ca6f102e662a2315aeff author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-05 20:05:44 +0300 message: Add an option to specify port number --- libmarias3/marias3.h | 3 ++- src/marias3.c | 14 ++++++++++++++ src/request.c | 3 +++ src/structs.h | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/libmarias3/marias3.h b/libmarias3/marias3.h index abe255b..a6f2bf3 100644 --- a/libmarias3/marias3.h +++ b/libmarias3/marias3.h @@ -80,7 +80,8 @@ enum ms3_set_option_t MS3_OPT_DISABLE_SSL_VERIFY, MS3_OPT_BUFFER_CHUNK_SIZE, MS3_OPT_FORCE_LIST_VERSION, - MS3_OPT_FORCE_PROTOCOL_VERSION + MS3_OPT_FORCE_PROTOCOL_VERSION, + MS3_OPT_PORT_NUMBER }; typedef enum ms3_set_option_t ms3_set_option_t; diff --git a/src/marias3.c b/src/marias3.c index 581b181..cebcbb0 100644 --- a/src/marias3.c +++ b/src/marias3.c @@ -166,6 +166,7 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret, ms3->s3key = ms3_cstrdup(s3key); ms3->s3secret = ms3_cstrdup(s3secret); ms3->region = ms3_cstrdup(region); + ms3->port = 0; /* The default value */ if (base_domain && strlen(base_domain)) { @@ -534,6 +535,19 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) break; } + case MS3_OPT_PORT_NUMBER: + { + int port_number; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + memcpy(&port_number, (void*)value, sizeof(int)); + + ms3->port = port_number; + break; + } default: return MS3_ERR_PARAMETER; } diff --git a/src/request.c b/src/request.c index 425367b..80a32d0 100644 --- a/src/request.c +++ b/src/request.c @@ -775,6 +775,9 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); } + if (ms3->port) + curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem); diff --git a/src/structs.h b/src/structs.h index 8d30a57..de80b53 100644 --- a/src/structs.h +++ b/src/structs.h @@ -42,6 +42,7 @@ struct ms3_st char *s3secret; char *region; char *base_domain; + int port; // 0 means "Use default" size_t buffer_chunk_size; CURL *curl; char *last_error;

1 0

[Commits] cd0e4f28fd4: MDEV-23662: Make S3 Storage Engine usable with MinIO
by Sergei Petrunia 03 Sep '20

03 Sep '20

revision-id: cd0e4f28fd45433e98f28933bc3ed103b34b64ac (mariadb-10.5.4-167-gcd0e4f28fd4) parent(s): 0af6e525212f81c2da8185817c6ca2e7c3ac036a author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-04 00:15:22 +0300 message: MDEV-23662: Make S3 Storage Engine usable with MinIO Add @@s3_port and @@s3_use_http. One can set these to access the local MinIO, for example. --- mysql-test/suite/s3/basic.result | 2 ++ mysql-test/suite/s3/my.cnf | 12 ++++++++++++ mysql-test/suite/s3/slave.cnf | 12 ++++++++++++ storage/maria/ha_s3.cc | 15 +++++++++++++++ storage/maria/s3_func.c | 6 ++++++ storage/maria/s3_func.h | 2 ++ 6 files changed, 49 insertions(+) diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index 9cddca79fc5..cf7d7377d92 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -94,11 +94,13 @@ s3_pagecache_age_threshold X s3_pagecache_buffer_size X s3_pagecache_division_limit X s3_pagecache_file_hash_size X +s3_port X s3_protocol_version X s3_region X s3_replicate_alter_as_create_select X s3_secret_key X s3_slave_ignore_updates X +s3_use_http X show variables like "s3_slave%"; Variable_name Value s3_slave_ignore_updates OFF diff --git a/mysql-test/suite/s3/my.cnf b/mysql-test/suite/s3/my.cnf index 6f8c27d4d0d..e810449d5b3 100644 --- a/mysql-test/suite/s3/my.cnf +++ b/mysql-test/suite/s3/my.cnf @@ -10,3 +10,15 @@ s3=ON #s3-access-key=... #s3-secret-key=... #s3-region=eu-north-1 + +## +## Configuration for local MinIO +## +s3-host-name="127.0.0.1" +# Note: s3-host-name="localhost" doesn't work. It causes +# libmarias3 to use the wrong variant of the protocol. +s3-bucket=storage-engine +s3-access-key=minioadmin +s3-secret-key=minioadmin +s3-port=9000 +s3-use-http=ON diff --git a/mysql-test/suite/s3/slave.cnf b/mysql-test/suite/s3/slave.cnf index 250a46e6322..c5e7f466226 100644 --- a/mysql-test/suite/s3/slave.cnf +++ b/mysql-test/suite/s3/slave.cnf @@ -6,3 +6,15 @@ s3=ON #s3-access-key=... #s3-secret-key=... #s3-region=eu-north-1 + +## +## Configuration for local MinIO +## +s3-host-name="127.0.0.1" +# Note: s3-host-name="localhost" doesn't work. It causes +# libmarias3 to use the wrong variant of the protocol. +s3-bucket=storage-engine +s3-access-key=minioadmin +s3-secret-key=minioadmin +s3-port=9000 +s3-use-http=ON diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index 1aae5a5b81f..190d99035a5 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -80,6 +80,8 @@ static ulong s3_pagecache_file_hash_size; static ulonglong s3_pagecache_buffer_size; static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; static char *s3_host_name; +static int s3_port; +static my_bool s3_use_http; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; static my_bool s3_debug= 0, s3_slave_ignore_updates= 0; static my_bool s3_replicate_alter_as_create_select= 0; @@ -181,6 +183,15 @@ static MYSQL_SYSVAR_STR(host_name, s3_host_name, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "AWS host name", 0, 0, DEFAULT_AWS_HOST_NAME); +static MYSQL_SYSVAR_INT(port, s3_port, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Port number to connect to (0 means use default)", + NULL /*check*/, NULL /*update*/, 0 /*default*/, + 0 /*min*/, 65535 /*max*/, 1 /*blk*/); +static MYSQL_SYSVAR_BOOL(use_http, s3_use_http, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "If true, force use of HTTP protocol", + NULL /*check*/, NULL /*update*/, 0 /*default*/); static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, "AWS access key", @@ -276,6 +287,8 @@ static my_bool s3_info_init(S3_INFO *info) return 1; info->protocol_version= (uint8_t) s3_protocol_version; lex_string_set(&info->host_name, s3_host_name); + info->port= s3_port; + info->use_http= s3_use_http; lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->region, s3_region); @@ -1050,6 +1063,8 @@ static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(pagecache_file_hash_size), MYSQL_SYSVAR(host_name), + MYSQL_SYSVAR(port), + MYSQL_SYSVAR(use_http), MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(secret_key), diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index bbaf048d534..9f40790a371 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -157,6 +157,12 @@ ms3_st *s3_open_connection(S3_INFO *s3) if (s3->protocol_version) ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, &s3->protocol_version); + if (s3->port) + ms3_set_option(s3_client, MS3_OPT_PORT_NUMBER, &s3->port); + + if (s3->use_http) + ms3_set_option(s3_client, MS3_OPT_USE_HTTP, NULL); + return s3_client; } diff --git a/storage/maria/s3_func.h b/storage/maria/s3_func.h index 8981517dd4d..bd5275494bc 100644 --- a/storage/maria/s3_func.h +++ b/storage/maria/s3_func.h @@ -45,6 +45,8 @@ typedef struct s3_info { /* Connection strings */ LEX_CSTRING access_key, secret_key, region, bucket, host_name; + int port; // 0 means 'Use default' + my_bool use_http; /* Will be set by caller or by ma_open() */ LEX_CSTRING database, table;

1 0

[Commits] 579b785: Add an option to specify port number
by Sergei Petrunia 03 Sep '20

03 Sep '20

revision-id: 579b785002a6a31ecf84d0391de7d839aa612af0 (libmarias3-3.1.2-3-g579b785) parent(s): d172e86c16224b4e0229ca6f102e662a2315aeff author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-04 00:01:08 +0300 message: Add an option to specify port number --- libmarias3/marias3.h | 3 ++- src/marias3.c | 14 ++++++++++++++ src/request.c | 34 +++++++++++++++++++++------------- src/structs.h | 1 + 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/libmarias3/marias3.h b/libmarias3/marias3.h index abe255b..a6f2bf3 100644 --- a/libmarias3/marias3.h +++ b/libmarias3/marias3.h @@ -80,7 +80,8 @@ enum ms3_set_option_t MS3_OPT_DISABLE_SSL_VERIFY, MS3_OPT_BUFFER_CHUNK_SIZE, MS3_OPT_FORCE_LIST_VERSION, - MS3_OPT_FORCE_PROTOCOL_VERSION + MS3_OPT_FORCE_PROTOCOL_VERSION, + MS3_OPT_PORT_NUMBER }; typedef enum ms3_set_option_t ms3_set_option_t; diff --git a/src/marias3.c b/src/marias3.c index 581b181..cebcbb0 100644 --- a/src/marias3.c +++ b/src/marias3.c @@ -166,6 +166,7 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret, ms3->s3key = ms3_cstrdup(s3key); ms3->s3secret = ms3_cstrdup(s3secret); ms3->region = ms3_cstrdup(region); + ms3->port = 0; /* The default value */ if (base_domain && strlen(base_domain)) { @@ -534,6 +535,19 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) break; } + case MS3_OPT_PORT_NUMBER: + { + int port_number; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + memcpy(&port_number, (void*)value, sizeof(int)); + + ms3->port = port_number; + break; + } default: return MS3_ERR_PARAMETER; } diff --git a/src/request.c b/src/request.c index 425367b..cd819c2 100644 --- a/src/request.c +++ b/src/request.c @@ -51,7 +51,7 @@ static void set_error_nocopy(ms3_st *ms3, char *error) ms3->last_error = error; } -static uint8_t build_request_uri(CURL *curl, const char *base_domain, +static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, const char *bucket, const char *object, const char *query, bool use_http, uint8_t protocol_version) { @@ -61,6 +61,14 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, const char *http_protocol = "http"; const char *https_protocol = "https"; const char *protocol; + char port_str[32]; // ":portnumber" or empty string + + if (port) + { + snprintf(port_str, sizeof(port_str) - 1, ":%d", port); + } + else + port_str[0]= 0; // empty string if (base_domain) { @@ -83,43 +91,43 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, if (query) { if (path_parts + strlen(domain) + strlen(bucket) + strlen(object) + strlen( - query) >= MAX_URI_LENGTH - 1) + query) + strlen(port_str) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s?%s", protocol, - domain, bucket, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s?%s", protocol, + domain, port_str, bucket, object, query); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s?%s", protocol, - bucket, domain, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s?%s", protocol, + bucket, domain, port_str, object, query); } } else { if (path_parts + strlen(domain) + strlen(bucket) + strlen( - object) >= MAX_URI_LENGTH - 1) + object) + strlen(port_str) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s", protocol, - domain, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s", protocol, + domain, port_str, bucket, object); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s", protocol, - bucket, domain, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s", protocol, + bucket, domain, port_str, object); } } @@ -717,8 +725,8 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, ms3->query_buffer); } - res = build_request_uri(curl, ms3->base_domain, bucket, path, query, - ms3->use_http, ms3->protocol_version); + res = build_request_uri(curl, ms3->base_domain, ms3->port, bucket, path, + query, ms3->use_http, ms3->protocol_version); if (res) { diff --git a/src/structs.h b/src/structs.h index 8d30a57..de80b53 100644 --- a/src/structs.h +++ b/src/structs.h @@ -42,6 +42,7 @@ struct ms3_st char *s3secret; char *region; char *base_domain; + int port; // 0 means "Use default" size_t buffer_chunk_size; CURL *curl; char *last_error;

1 0

[Commits] 920c8ff: Merge branch 'xpand-beta10-replication'
by Sergei Petrunia 01 Sep '20

01 Sep '20

revision-id: 920c8ff30d76ca2c5a9befc7d369180099ce5164 () parent(s): c97e3d5147a15771c30a48233cdf18a5305ada52 edb119b5d16529060f86eb75411ce4e07db517ae author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-01 19:50:18 +0300 message: Merge branch 'xpand-beta10-replication' ha_xpand.cc | 20 ++++++++++++++------ mysql-test/xpand/rpl_basic.result | 21 ++++++++++++++++++++- mysql-test/xpand/rpl_basic.test | 26 ++++++++++++++++++++++++-- mysql-test/xpand/xpand_plugin.result | 2 +- 4 files changed, 59 insertions(+), 10 deletions(-)

1 0

[Commits] 41d807dfc8c: MENT-731:Setting of SHUTDOWN_WAIT_FOR_SLAVES should require special privilege, not generic SUPER
by sujatha 31 Aug '20

31 Aug '20

revision-id: 41d807dfc8c5f640ccbbac5838cf572b2b1712c1 (mariadb-10.4.4-1804-g41d807dfc8c) parent(s): 38cbb0915eb4a0c43f1a83afa37c7de5e481a718 author: Sujatha committer: Sujatha timestamp: 2020-08-31 22:30:38 +0530 message: MENT-731:Setting of SHUTDOWN_WAIT_FOR_SLAVES should require special privilege, not generic SUPER Fix: === 'shutdown_wait_for_slaves' variable can be set either by having SUPER or SHUTDOWN privilege. --- mysql-test/suite/binlog/r/binlog_grant.result | 34 +++++++++++++++++++++++ mysql-test/suite/binlog/t/binlog_grant.test | 40 +++++++++++++++++++++++++++ sql/privilege.h | 3 ++ sql/sys_vars.cc | 5 +++- 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/binlog/r/binlog_grant.result b/mysql-test/suite/binlog/r/binlog_grant.result index af4c02411d5..aee0730c0ab 100644 --- a/mysql-test/suite/binlog/r/binlog_grant.result +++ b/mysql-test/suite/binlog/r/binlog_grant.result @@ -124,6 +124,40 @@ disconnect user1; connection default; DROP USER user1@localhost; # +# MENT-731: Setting of SHUTDOWN_WAIT_FOR_SLAVES should require special +# privilege, not generic SUPER +# +# Test that 'shutdown_wait_for_slaves' variable cannot be set without +# SHUTDOWN or SUPER +CREATE USER user1@localhost; +GRANT ALL PRIVILEGES ON *.* TO user1@localhost; +REVOKE SHUTDOWN, SUPER ON *.* FROM user1@localhost; +connect user1,localhost,user1,,; +connection user1; +SET GLOBAL SHUTDOWN_WAIT_FOR_SLAVES=ON; +ERROR 42000: Access denied; you need (at least one of) the SHUTDOWN, SUPER privilege(s) for this operation +disconnect user1; +connection default; +DROP USER user1@localhost; +# Test that 'shutdown_wait_for_slaves' variable can be set with SHUTDOWN +CREATE USER user1@localhost; +GRANT SHUTDOWN ON *.* TO user1@localhost; +connect user1,localhost,user1,,; +connection user1; +SET GLOBAL SHUTDOWN_WAIT_FOR_SLAVES=ON; +disconnect user1; +connection default; +DROP USER user1@localhost; +# Test that 'shutdown_wait_for_slaves' variable can be set with SUPER +CREATE USER user1@localhost; +GRANT SUPER ON *.* TO user1@localhost; +connect user1,localhost,user1,,; +connection user1; +SET GLOBAL SHUTDOWN_WAIT_FOR_SLAVES=OFF; +disconnect user1; +connection default; +DROP USER user1@localhost; +# # MDEV-21975 Add BINLOG REPLAY privilege and bind new privileges to # gtid_seq_no, preudo_thread_id, server_id, gtid_domain_id # diff --git a/mysql-test/suite/binlog/t/binlog_grant.test b/mysql-test/suite/binlog/t/binlog_grant.test index ec8d2aeccd2..ca0e40299a9 100644 --- a/mysql-test/suite/binlog/t/binlog_grant.test +++ b/mysql-test/suite/binlog/t/binlog_grant.test @@ -172,6 +172,46 @@ SHOW BINLOG EVENTS; connection default; DROP USER user1@localhost; +--echo # +--echo # MENT-731: Setting of SHUTDOWN_WAIT_FOR_SLAVES should require special +--echo # privilege, not generic SUPER +--echo # + +--echo # Test that 'shutdown_wait_for_slaves' variable cannot be set without +--echo # SHUTDOWN or SUPER +CREATE USER user1@localhost; +GRANT ALL PRIVILEGES ON *.* TO user1@localhost; +REVOKE SHUTDOWN, SUPER ON *.* FROM user1@localhost; +--connect(user1,localhost,user1,,) +--connection user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +SET GLOBAL SHUTDOWN_WAIT_FOR_SLAVES=ON; +--disconnect user1 +--connection default +DROP USER user1@localhost; + + +--echo # Test that 'shutdown_wait_for_slaves' variable can be set with SHUTDOWN +CREATE USER user1@localhost; +GRANT SHUTDOWN ON *.* TO user1@localhost; +--connect(user1,localhost,user1,,) +--connection user1 +SET GLOBAL SHUTDOWN_WAIT_FOR_SLAVES=ON; +--disconnect user1 +connection default; +DROP USER user1@localhost; + + +--echo # Test that 'shutdown_wait_for_slaves' variable can be set with SUPER +CREATE USER user1@localhost; +GRANT SUPER ON *.* TO user1@localhost; +--connect(user1,localhost,user1,,) +--connection user1 +SET GLOBAL SHUTDOWN_WAIT_FOR_SLAVES=OFF; +--disconnect user1 +connection default; +DROP USER user1@localhost; + --echo # --echo # MDEV-21975 Add BINLOG REPLAY privilege and bind new privileges to --echo # gtid_seq_no, preudo_thread_id, server_id, gtid_domain_id diff --git a/sql/privilege.h b/sql/privilege.h index 37cdf4da01a..f4bfd835013 100644 --- a/sql/privilege.h +++ b/sql/privilege.h @@ -433,6 +433,9 @@ constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_SLOW_LAUNCH_TIME= constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_THREAD_POOL= CONNECTION_ADMIN_ACL | SUPER_ACL; +// Was SUPER_ACL prior to 10.5.5 +constexpr privilege_t PRIV_SET_SYSTEM_VAR_SHUTDOWN_WAIT_FOR_SLAVES= + SHUTDOWN_ACL | SUPER_ACL; /* Binary log related privileges that are checked regardless diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 1577457a5d7..396c0570215 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -4573,7 +4573,10 @@ static Sys_var_ulong Sys_profiling_history_size( VALID_RANGE(0, 100), DEFAULT(15), BLOCK_SIZE(1)); #endif -static Sys_var_mybool Sys_shutdown_wait_for_slaves( +static Sys_var_on_access_global<Sys_var_mybool, + PRIV_SET_SYSTEM_VAR_SHUTDOWN_WAIT_FOR_SLAVES> + +Sys_shutdown_wait_for_slaves( "shutdown_wait_for_slaves", "when ON, SHUTDOWN command runs with implicit WAIT FOR ALL SLAVES option.", GLOBAL_VAR(opt_shutdown_wait_for_slaves), CMD_LINE(OPT_ARG),

1 0

[Commits] e4fa0c805ad: MDEV-4633: multi_source.simple test fails sporadically
by sujatha 28 Aug '20

28 Aug '20

revision-id: e4fa0c805adf06c637cc3867427ca529b40e19ce (mariadb-10.1.43-267-ge4fa0c805ad) parent(s): de49e603770463e64cf4a79a7414bac93c02254f author: Sujatha committer: Sujatha timestamp: 2020-08-28 12:03:09 +0530 message: MDEV-4633: multi_source.simple test fails sporadically Analysis: ======== Writes to 'rli->log_space_total' needs to be synchronized. All modifications to 'rli->log_space_total' should be performed by holding 'rli->log_space_lock'. But IO thread doesn't acquire this mutex while writing events to relay log. Hence there is a race condition between SQL THREAD and IO THREAD, when both try to modify the same variable. SQL_THREAD is trying to decrement 'rli->log_space_total' in 'purge_first_log' and IO_THREAD is trying to increment the 'rli->log_space_total' in 'queue_event' simultaneously . Hence test occasionally fails with result mismatch. Fix: === Acquire 'rli->log_space_lock' while modifying 'rli->log_space_total'. --- sql/log.cc | 15 +++++++++++++++ sql/log.h | 13 +------------ sql/rpl_rli.cc | 4 ++++ sql/slave.cc | 28 ++++++++++++++-------------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index 8049b94bab1..28a3d58d9e4 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -10265,6 +10265,21 @@ MYSQL_BIN_LOG::do_binlog_recovery(const char *opt_name, bool do_xa_recovery) return error; } +void MYSQL_BIN_LOG::harvest_bytes_written(Relay_log_info *rli) +{ +#ifndef DBUG_OFF + char buf1[22],buf2[22]; +#endif + DBUG_ENTER("harvest_bytes_written"); + mysql_mutex_lock(&rli->log_space_lock); + rli->log_space_total+=bytes_written; + DBUG_PRINT("info",("relay_log_space: %s bytes_written: %s", + llstr(rli->log_space_total,buf1), + llstr(bytes_written,buf2))); + bytes_written=0; + mysql_mutex_unlock(&rli->log_space_lock); + DBUG_VOID_RETURN; +} #ifdef INNODB_COMPATIBILITY_HOOKS /** diff --git a/sql/log.h b/sql/log.h index 277e5c6f69c..3b136c47bcf 100644 --- a/sql/log.h +++ b/sql/log.h @@ -706,18 +706,7 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG { bytes_written = 0; } - void harvest_bytes_written(ulonglong* counter) - { -#ifndef DBUG_OFF - char buf1[22],buf2[22]; -#endif - DBUG_ENTER("harvest_bytes_written"); - (*counter)+=bytes_written; - DBUG_PRINT("info",("counter: %s bytes_written: %s", llstr(*counter,buf1), - llstr(bytes_written,buf2))); - bytes_written=0; - DBUG_VOID_RETURN; - } + void harvest_bytes_written(Relay_log_info *rli); void set_max_size(ulong max_size_arg); void signal_update(); void wait_for_sufficient_commits(); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index c196a65809a..18d7db234fe 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -467,7 +467,9 @@ static inline int add_relay_log(Relay_log_info* rli,LOG_INFO* linfo) linfo->log_file_name); DBUG_RETURN(1); } + mysql_mutex_lock(&rli->log_space_lock); rli->log_space_total += s.st_size; + mysql_mutex_unlock(&rli->log_space_lock); DBUG_PRINT("info",("log_space_total: %llu", rli->log_space_total)); DBUG_RETURN(0); } @@ -477,7 +479,9 @@ static int count_relay_log_space(Relay_log_info* rli) { LOG_INFO linfo; DBUG_ENTER("count_relay_log_space"); + mysql_mutex_lock(&rli->log_space_lock); rli->log_space_total= 0; + mysql_mutex_unlock(&rli->log_space_lock); if (rli->relay_log.find_log_pos(&linfo, NullS, 1)) { sql_print_error("Could not find first log while counting relay log space"); diff --git a/sql/slave.cc b/sql/slave.cc index 06f2b0d955a..3d9e23441bc 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2508,7 +2508,7 @@ static void write_ignored_events_info_to_relay_log(THD *thd, Master_info *mi) } if (likely (rev || glev)) { - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); if (flush_master_info(mi, TRUE, TRUE)) sql_print_error("Failed to flush master info file"); } @@ -5270,7 +5270,7 @@ static int process_io_create_file(Master_info* mi, Create_file_log_event* cev) "error writing Exec_load event to relay log"); goto err; } - mi->rli.relay_log.harvest_bytes_written(&mi->rli.log_space_total); + mi->rli.relay_log.harvest_bytes_written(&mi->rli); break; } if (unlikely(cev_not_written)) @@ -5285,7 +5285,7 @@ static int process_io_create_file(Master_info* mi, Create_file_log_event* cev) goto err; } cev_not_written=0; - mi->rli.relay_log.harvest_bytes_written(&mi->rli.log_space_total); + mi->rli.relay_log.harvest_bytes_written(&mi->rli); } else { @@ -5299,7 +5299,7 @@ static int process_io_create_file(Master_info* mi, Create_file_log_event* cev) "error writing Append_block event to relay log"); goto err; } - mi->rli.relay_log.harvest_bytes_written(&mi->rli.log_space_total) ; + mi->rli.relay_log.harvest_bytes_written(&mi->rli) ; } } } @@ -5489,7 +5489,7 @@ static int queue_binlog_ver_1_event(Master_info *mi, const char *buf, mysql_mutex_unlock(&mi->data_lock); DBUG_RETURN(1); } - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); } delete ev; mi->master_log_pos+= inc_pos; @@ -5547,7 +5547,7 @@ static int queue_binlog_ver_3_event(Master_info *mi, const char *buf, mysql_mutex_unlock(&mi->data_lock); DBUG_RETURN(1); } - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); delete ev; mi->master_log_pos+= inc_pos; err: @@ -5784,9 +5784,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) mysql_mutex_lock(log_lock); if (likely(!rli->relay_log.write_event(&fdle) && !rli->relay_log.flush_and_sync(NULL))) - { - rli->relay_log.harvest_bytes_written(&rli->log_space_total); - } + rli->relay_log.harvest_bytes_written(rli); else { error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; @@ -6221,7 +6219,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) (rli->relay_log.description_event_for_queue)) error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; else - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); } else if (mi->gtid_reconnect_event_skip_count == 0) { @@ -6234,7 +6232,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) if (rli->relay_log.append_no_lock(&fake_rev)) error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; else - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); } } else @@ -6302,7 +6300,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) { mi->master_log_pos+= inc_pos; DBUG_PRINT("info", ("master_log_pos: %lu", (ulong) mi->master_log_pos)); - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); } else { @@ -6382,7 +6380,7 @@ void end_relay_log_info(Relay_log_info* rli) log_lock= rli->relay_log.get_log_lock(); mysql_mutex_lock(log_lock); rli->relay_log.close(LOG_CLOSE_INDEX | LOG_CLOSE_STOP_EVENT); - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + rli->relay_log.harvest_bytes_written(rli); mysql_mutex_unlock(log_lock); /* Delete the slave's temporary tables from memory. @@ -7318,7 +7316,9 @@ int rotate_relay_log(Master_info* mi) Note that it needs to be protected by mi->data_lock. */ mysql_mutex_assert_owner(&mi->data_lock); - rli->relay_log.harvest_bytes_written(&rli->log_space_total); + + rli->relay_log.harvest_bytes_written(rli); + end: DBUG_RETURN(error); }

1 0