revision-id: 9c89deac57e27d2564a3c2986d548f4f587ca357 (mariadb-10.4.1-159-g9c89deac57e) parent(s): eb1d7aeeea2978f5be10cf95e6f638f184cfe4ad author: Andrei Elkin committer: Andrei Elkin timestamp: 2019-02-26 20:55:08 +0200 message: MDEV-18450 Slaves wait shutdown The patches features an optional shutdown behavior to hold on until after all connected slaves have been sent the last binlogged event. The connected slave is one whose START SLAVE has been acknowledged and that was not stopped since that though it could be technically reconnecting in background. The solution therefore disallows killing the dump thread until is has found EOF of the latest binlog file. It is up to the shutdown requester (DBA) to set up a sufficiently large shutdown timeout value for shudown to wait patiently until lagging behind slaves have been synchronized. On the other hand if a specific slave needs exclusion from synchronization the DBA would have to stop it manually which would terminate its dump thread. `mysqladmin shutdown' is extended with a `--wait_for_slaves' option which translates to `SHUTDOWN wait_for_slaves' sql query to enable the feature on the client side. --- client/client_priv.h | 1 + client/mysqladmin.cc | 20 +++- .../suite/rpl/include/rpl_shutdown_wait_slaves.inc | 82 +++++++++++++ .../rpl/r/rpl_shutdown_wait_semisync_slaves.result | 58 +++++++++ .../suite/rpl/r/rpl_shutdown_wait_slaves.result | 35 ++++++ .../rpl/t/rpl_shutdown_wait_semisync_slaves.cnf | 17 +++ .../rpl/t/rpl_shutdown_wait_semisync_slaves.test | 46 ++++++++ .../suite/rpl/t/rpl_shutdown_wait_slaves.cnf | 17 +++ .../suite/rpl/t/rpl_shutdown_wait_slaves.test | 11 ++ sql/log.cc | 2 +- sql/log.h | 10 ++ sql/mysqld.cc | 131 ++++++++++++++++++++- sql/mysqld.h | 4 +- sql/repl_failsafe.cc | 4 + sql/sql_class.cc | 2 +- sql/sql_class.h | 1 + sql/sql_lex.h | 1 + sql/sql_parse.cc | 1 + sql/sql_repl.cc | 17 ++- sql/sql_yacc.yy | 9 ++ sql/sql_yacc_ora.yy | 8 ++ 21 files changed, 464 insertions(+), 13 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index ffdf0fc4b8f..54fed943313 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -100,6 +100,7 @@ enum options_client OPT_SKIP_ANNOTATE_ROWS_EVENTS, OPT_SSL_CRL, OPT_SSL_CRLPATH, OPT_PRINT_ROW_COUNT, OPT_PRINT_ROW_EVENT_POSITIONS, + OPT_SHUTDOWN_WAIT_FOR_SLAVES, OPT_MAX_CLIENT_OPTION /* should be always the last */ }; diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index 35afe06eade..dc0eec3482e 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -40,7 +40,8 @@ ulonglong last_values[MAX_MYSQL_VAR+100]; static int interval=0; static my_bool option_force=0,interrupted=0,new_line=0, opt_compress= 0, opt_local= 0, opt_relative= 0, opt_verbose= 0, - opt_vertical= 0, tty_password= 0, opt_nobeep; + opt_vertical= 0, tty_password= 0, opt_nobeep, + opt_shutdown_wait_for_slaves= 0; static my_bool debug_info_flag= 0, debug_check_flag= 0; static uint tcp_port = 0, option_wait = 0, option_silent=0, nr_iterations; static uint opt_count_iterations= 0, my_end_arg; @@ -218,6 +219,11 @@ static struct my_option my_long_options[] = {"shutdown_timeout", OPT_SHUTDOWN_TIMEOUT, "", &opt_shutdown_timeout, &opt_shutdown_timeout, 0, GET_ULONG, REQUIRED_ARG, SHUTDOWN_DEF_TIMEOUT, 0, 3600*12, 0, 1, 0}, + {"wait_for_slaves", OPT_SHUTDOWN_WAIT_FOR_SLAVES, + "Defers shutdown until after all binlogged events have been sent to " + "all connected slaves", &opt_shutdown_wait_for_slaves, + &opt_shutdown_wait_for_slaves, 0, GET_BOOL, NO_ARG, 0, 0, 0, + 0, 0, 0}, {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.", &opt_plugin_dir, &opt_plugin_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -693,7 +699,17 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) !stat(pidfile, &pidfile_status)) last_modified= pidfile_status.st_mtime; - if (mysql_shutdown(mysql, SHUTDOWN_DEFAULT)) + if (opt_shutdown_wait_for_slaves) + { + sprintf(buff, "SHUTDOWN WAIT FOR SLAVES"); + if (mysql_query(mysql, buff)) + { + my_printf_error(0, "%s failed; error: '%-.200s'", + error_flags, buff, mysql_error(mysql)); + return -1; + } + } + else if (mysql_shutdown(mysql, SHUTDOWN_DEFAULT)) { my_printf_error(0, "shutdown failed; error: '%s'", error_flags, mysql_error(mysql)); diff --git a/mysql-test/suite/rpl/include/rpl_shutdown_wait_slaves.inc b/mysql-test/suite/rpl/include/rpl_shutdown_wait_slaves.inc new file mode 100644 index 00000000000..4e6f38150b5 --- /dev/null +++ b/mysql-test/suite/rpl/include/rpl_shutdown_wait_slaves.inc @@ -0,0 +1,82 @@ +--connection server_1 + +call mtr.add_suppression("Dump thread [0-9]+ last sent to server [0-9]+ binlog file:pos .+"); +CREATE TABLE t1 (a INT) ENGINE=innodb; + +--save_master_pos + +--connection server_2 +--sync_with_master + +--connection server_3 +--sync_with_master + +--connection server_4 +--source include/stop_slave.inc + +--connection server_1 +--disable_query_log +--let $count=1000 +while ($count) +{ + INSERT INTO t1 SET a=1; + --dec $count +} +--enable_query_log +--save_master_pos + +# Shutdown master and restart server_4 who will be waiting for the master +# to start replication at its shutdown beginning phase. +# The being forked out server_4 dump thread must relate to a record +# in slave_list, and it won't start sending out binlog events +# until has received a signal from the shutdown thread. +# This also proves delivery to a started-in-middle-of-shutdown slave. +--connection server_1 +SET @@GLOBAL.debug_dbug="+d,simulate_delay_at_shutdown"; + +--connection server_4 +--source include/start_slave.inc + +--connection server_1 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF +--shutdown_server 60 +--source include/wait_until_disconnected.inc +# +# MDEV-18450 liveness condition: +# Despite shutdown even "late" slave #4 is in sync +# +--connection server_4 +--sync_with_master + +--connection server_3 +--sync_with_master + +--connection server_2 +--sync_with_master + +--connection server_1 +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +--connection default +--enable_reconnect +--source include/wait_until_connected_again.inc +--connection server_1 +--enable_reconnect +--source include/wait_until_connected_again.inc + +# +# Cleanup +# +--connection server_1 +DROP TABLE t1; + +--connection server_2 +--source include/start_slave.inc +--connection server_3 +--source include/start_slave.inc +--connection server_4 +--source include/start_slave.inc diff --git a/mysql-test/suite/rpl/r/rpl_shutdown_wait_semisync_slaves.result b/mysql-test/suite/rpl/r/rpl_shutdown_wait_semisync_slaves.result new file mode 100644 index 00000000000..636d4988809 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_shutdown_wait_semisync_slaves.result @@ -0,0 +1,58 @@ +include/rpl_init.inc [topology=1->2, 1->3, 1->4] +connection server_1; +call mtr.add_suppression("Timeout waiting for reply of binlog"); +SET @@GLOBAL.rpl_semi_sync_master_enabled = 1; +connection server_2; +set global rpl_semi_sync_slave_enabled = 1; +include/stop_slave.inc +include/start_slave.inc +set global rpl_semi_sync_slave_enabled = 1; +connection server_3; +set global rpl_semi_sync_slave_enabled = 1; +include/stop_slave.inc +include/start_slave.inc +set global rpl_semi_sync_slave_enabled = 1; +connection server_1; +call mtr.add_suppression("Dump thread [0-9]+ last sent to server [0-9]+ binlog file:pos .+"); +CREATE TABLE t1 (a INT) ENGINE=innodb; +connection server_2; +connection server_3; +connection server_4; +include/stop_slave.inc +connection server_1; +connection server_1; +SET @@GLOBAL.debug_dbug="+d,simulate_delay_at_shutdown"; +connection server_4; +include/start_slave.inc +connection server_1; +connection server_4; +connection server_3; +connection server_2; +connection server_1; +connection default; +connection server_1; +connection server_1; +DROP TABLE t1; +connection server_2; +include/start_slave.inc +Warnings: +Note 1254 Slave is already running +connection server_3; +include/start_slave.inc +Warnings: +Note 1254 Slave is already running +connection server_4; +include/start_slave.inc +Warnings: +Note 1254 Slave is already running +connection server_2; +include/stop_slave.inc +include/start_slave.inc +SET @@GLOBAL.rpl_semi_sync_slave_enabled = 0;; +connection server_3; +include/stop_slave.inc +include/start_slave.inc +SET @@GLOBAL.rpl_semi_sync_slave_enabled = 0;; +connection server_1; +SET @@GLOBAL.rpl_semi_sync_master_enabled = 0; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_shutdown_wait_slaves.result b/mysql-test/suite/rpl/r/rpl_shutdown_wait_slaves.result new file mode 100644 index 00000000000..d0513b499e8 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_shutdown_wait_slaves.result @@ -0,0 +1,35 @@ +include/rpl_init.inc [topology=1->2, 1->3, 1->4] +connection server_1; +call mtr.add_suppression("Dump thread [0-9]+ last sent to server [0-9]+ binlog file:pos .+"); +CREATE TABLE t1 (a INT) ENGINE=innodb; +connection server_2; +connection server_3; +connection server_4; +include/stop_slave.inc +connection server_1; +connection server_1; +SET @@GLOBAL.debug_dbug="+d,simulate_delay_at_shutdown"; +connection server_4; +include/start_slave.inc +connection server_1; +connection server_4; +connection server_3; +connection server_2; +connection server_1; +connection default; +connection server_1; +connection server_1; +DROP TABLE t1; +connection server_2; +include/start_slave.inc +Warnings: +Note 1254 Slave is already running +connection server_3; +include/start_slave.inc +Warnings: +Note 1254 Slave is already running +connection server_4; +include/start_slave.inc +Warnings: +Note 1254 Slave is already running +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.cnf b/mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.cnf new file mode 100644 index 00000000000..032e6081acc --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.cnf @@ -0,0 +1,17 @@ +!include ../my.cnf + +[mysqld.1] +debug_dbug="+d,mysql_admin_slow_shutdown" +gdb +[mysqld.2] + +[mysqld.3] + +[mysqld.4] + +[ENV] +SERVER_MYPORT_3= @mysqld.3.port +SERVER_MYSOCK_3= @mysqld.3.socket + +SERVER_MYPORT_4= @mysqld.4.port +SERVER_MYSOCK_4= @mysqld.4.socket diff --git a/mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.test b/mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.test new file mode 100644 index 00000000000..2c63df30fde --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.test @@ -0,0 +1,46 @@ +# +# MDEV-18450 "Slow" shutdown to wait for slaves that are to be fed +# with everything in the master binlog before shutdown completes. +# +# This is a semisync version of basic tests. +--source include/have_innodb.inc +--source include/have_debug.inc +--let $rpl_topology=1->2, 1->3, 1->4 +--source include/rpl_init.inc + +--connection server_1 +call mtr.add_suppression("Timeout waiting for reply of binlog"); +--let $sav_enabled_master=`SELECT @@GLOBAL.rpl_semi_sync_master_enabled` +SET @@GLOBAL.rpl_semi_sync_master_enabled = 1; + +--let slaves= 3 +--let i= 2 +while (`SELECT $i <= $slaves`) +{ + --connection server_$i + --let $sav_enabled_slave=`SELECT @@GLOBAL.rpl_semi_sync_slave_enabled` + set global rpl_semi_sync_slave_enabled = 1; + + source include/stop_slave.inc; + source include/start_slave.inc; + set global rpl_semi_sync_slave_enabled = 1; + + --inc $i +} + +--source include/rpl_shutdown_wait_slaves.inc +--let i= 2 +while (`SELECT $i <= $slaves`) +{ + --connection server_$i + source include/stop_slave.inc; + source include/start_slave.inc; + --eval SET @@GLOBAL.rpl_semi_sync_slave_enabled = $sav_enabled_slave; + + --inc $i +} + +--connection server_1 +--eval SET @@GLOBAL.rpl_semi_sync_master_enabled = $sav_enabled_master + +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.cnf b/mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.cnf new file mode 100644 index 00000000000..032e6081acc --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.cnf @@ -0,0 +1,17 @@ +!include ../my.cnf + +[mysqld.1] +debug_dbug="+d,mysql_admin_slow_shutdown" +gdb +[mysqld.2] + +[mysqld.3] + +[mysqld.4] + +[ENV] +SERVER_MYPORT_3= @mysqld.3.port +SERVER_MYSOCK_3= @mysqld.3.socket + +SERVER_MYPORT_4= @mysqld.4.port +SERVER_MYSOCK_4= @mysqld.4.socket diff --git a/mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.test b/mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.test new file mode 100644 index 00000000000..97363206776 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.test @@ -0,0 +1,11 @@ +# +# MDEV-18450 "Slow" shutdown to wait for slaves that are to be fed +# with everything in the master binlog before shutdown completes. +# +--source include/have_innodb.inc +--source include/have_debug.inc +--let $rpl_topology=1->2, 1->3, 1->4 +--source include/rpl_init.inc + +--source include/rpl_shutdown_wait_slaves.inc +--source include/rpl_end.inc diff --git a/sql/log.cc b/sql/log.cc index a4a2b4b1e37..e2870294fe5 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3222,7 +3222,7 @@ MYSQL_BIN_LOG::MYSQL_BIN_LOG(uint *sync_period) checksum_alg_reset(BINLOG_CHECKSUM_ALG_UNDEF), relay_log_checksum_alg(BINLOG_CHECKSUM_ALG_UNDEF), description_event_for_exec(0), description_event_for_queue(0), - current_binlog_id(0) + current_binlog_id(0), slaves_wait_shutdown(SHDN_NONE) { /* We don't want to initialize locks here as such initialization depends on diff --git a/sql/log.h b/sql/log.h index 2d4cc7a74a1..ca1aa268ae1 100644 --- a/sql/log.h +++ b/sql/log.h @@ -653,6 +653,16 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG char last_commit_pos_file[FN_REFLEN]; my_off_t last_commit_pos_offset; ulong current_binlog_id; + /* + to facilitate in running slow shutdown the var is changed from default + at shutdown, and is checked by dump threads. + */ + enum enum_shutdown_phase + { + SHDN_NONE= 0, + SHDN_PREPARE, + SHDN_DOIT + } slaves_wait_shutdown; MYSQL_BIN_LOG(uint *sync_period); /* diff --git a/sql/mysqld.cc b/sql/mysqld.cc index d4ca35b36cc..ead5e31f05b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -736,6 +736,7 @@ mysql_rwlock_t LOCK_ssl_refresh; mysql_rwlock_t LOCK_all_status_vars; mysql_prlock_t LOCK_system_variables_hash; mysql_cond_t COND_thread_count, COND_start_thread; +mysql_cond_t COND_slave_list; pthread_t signal_thread; pthread_attr_t connection_attrib; mysql_mutex_t LOCK_server_started; @@ -1038,7 +1039,7 @@ PSI_cond_key key_BINLOG_COND_xid_list, key_TABLE_SHARE_cond, key_user_level_lock_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, key_COND_start_thread, key_COND_binlog_send, - key_BINLOG_COND_queue_busy; + key_BINLOG_COND_queue_busy, key_COND_slave_list; PSI_cond_key key_RELAYLOG_COND_relay_log_updated, key_RELAYLOG_COND_bin_log_updated, key_COND_wakeup_ready, key_COND_wait_commit; @@ -1102,7 +1103,8 @@ static PSI_cond_info all_server_conds[]= { &key_COND_gtid_ignore_duplicates, "COND_gtid_ignore_duplicates", 0}, { &key_COND_ack_receiver, "Ack_receiver::cond", 0}, { &key_COND_binlog_send, "COND_binlog_send", 0}, - { &key_TABLE_SHARE_COND_rotation, "TABLE_SHARE::COND_rotation", 0} + { &key_TABLE_SHARE_COND_rotation, "TABLE_SHARE::COND_rotation", 0}, + { &key_COND_slave_list, "COND_slave_list", PSI_FLAG_GLOBAL} }; PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert, @@ -1526,13 +1528,36 @@ static void end_ssl(); ** Code to end mysqld ****************************************************************************/ -static my_bool kill_all_threads(THD *thd, void *) + +static bool is_dump_thread_no_lock(THD *thd) +{ + + DBUG_ASSERT(!thd->rpl_dump_thread || + my_hash_search(&slave_list, (uchar*) &thd->variables.server_id, 4)); + return thd->rpl_dump_thread; +} + +static bool is_dump_thread(THD *thd) +{ + bool rc; + + mysql_mutex_lock(&LOCK_slave_list); + rc= is_dump_thread_no_lock(thd); + mysql_mutex_unlock(&LOCK_slave_list); + + return rc; +} + +static my_bool kill_all_threads(THD *thd, void * ) { DBUG_PRINT("quit", ("Informing thread %ld that it's time to die", (ulong) thd->thread_id)); /* We skip slave threads on this first loop through. */ if (thd->slave_thread) return 0; + /* We skip master threads */ + if (is_dump_thread(thd)) + return 0; if (DBUG_EVALUATE_IF("only_kill_system_threads", !thd->system_thread, 0)) return 0; @@ -1576,6 +1601,12 @@ static my_bool kill_all_threads(THD *thd, void *) static my_bool kill_all_threads_once_again(THD *thd, void *) { #ifndef __bsdi__ // Bug in BSDI kernel + /* e.g "slow" shutdown leaves out dump threads */ + + /* We still skip dump threads */ + if (is_dump_thread(thd)) + return 0; + if (thd->vio_ok()) { if (global_system_variables.log_warnings) @@ -1625,6 +1656,45 @@ static my_bool kill_all_threads_once_again(THD *thd, void *) return 0; } +static my_bool kill_all_dump_threads(THD *thd, void *) +{ + /* e.g "slow" shutdown leaves out dump threads */ + + if (!is_dump_thread(thd)) + return 0; + + if (thd->vio_ok()) + { + if (global_system_variables.log_warnings) + sql_print_warning(ER_DEFAULT(ER_FORCING_CLOSE), my_progname, + (ulong) thd->thread_id, + (thd->main_security_ctx.user ? + thd->main_security_ctx.user : "")); + /* + close_connection() might need a valid current_thd + for memory allocation tracking. + */ + THD *save_thd= current_thd; + set_current_thd(thd); + close_connection(thd); + set_current_thd(save_thd); + } + return 0; +} + +static my_bool dump_thread_report_status(THD *thd, void *) +{ + // dump thread may not have yet current_linfo set + if (is_dump_thread_no_lock(thd)) + sql_print_warning("Dump thread %llu last sent to server %lu " + "binlog file:pos %s:%llu", + thd->thread_id, thd->variables.server_id, + thd->current_linfo ? + my_basename(thd->current_linfo->log_file_name) : "NULL", + thd->current_linfo ? thd->current_linfo->pos : 0); + return 0; +} + /** Kills main thread. @@ -1705,7 +1775,54 @@ void kill_mysql(THD *thd) { my_free(user); } - break_connect_loop(); + if (DBUG_EVALUATE_IF("mysql_admin_slow_shutdown", 1, + thd->lex->is_slaves_wait_shutdown)) + { + mysql_bin_log.slaves_wait_shutdown= MYSQL_BIN_LOG::SHDN_PREPARE; + debug_sync_set_action(thd, + STRING_WITH_LEN("now SIGNAL wait_for_done_waiting")); + DBUG_EXECUTE_IF("simulate_delay_at_shutdown", + { + DBUG_ASSERT(slave_list.records == 3); + const char act[]= + "now " + "SIGNAL greetings_from_kill_mysql"; + DBUG_ASSERT(!debug_sync_set_action(thd, + STRING_WITH_LEN(act))); + };); + break_connect_loop(); + } +} + + +void end_dump_threads() +{ + if (mysql_bin_log.slaves_wait_shutdown == MYSQL_BIN_LOG::SHDN_PREPARE) + { + mysql_bin_log.lock_binlog_end_pos(); + // set a mark for dump threads to prepare their exiting + mysql_bin_log.slaves_wait_shutdown= MYSQL_BIN_LOG::SHDN_DOIT; + // wakeup eager ones + mysql_bin_log.signal_bin_log_update(); + mysql_bin_log.unlock_binlog_end_pos(); + // wait for the rest slow ones + + mysql_mutex_lock(&LOCK_slave_list); + while (slave_list.records > 0) + { + struct timespec ts; + set_timespec_nsec(ts, 60); + + if (global_system_variables.log_warnings > 1) + server_threads.iterate(dump_thread_report_status); + mysql_cond_timedwait(&COND_slave_list, &LOCK_slave_list, &ts); + } + mysql_mutex_unlock(&LOCK_slave_list); + } + else + { + server_threads.iterate(kill_all_dump_threads); + } } @@ -1771,7 +1888,7 @@ static void close_connections(void) */ DBUG_PRINT("info", ("thread_count: %u", uint32_t(thread_count))); - for (int i= 0; thread_count && i < 1000; i++) + for (int i= 0; thread_count && i < 1000; i++) // TODO: MDEV-18450 wait for "external" fixes and then see if thread_count needs slave_list.records subtraction. my_sleep(20000); /* @@ -1780,7 +1897,7 @@ static void close_connections(void) client on a blocking read call are aborted. */ server_threads.iterate(kill_all_threads_once_again); - + end_dump_threads(); end_slave(); #ifdef WITH_WSREP if (wsrep_inited == 1) @@ -2130,6 +2247,7 @@ static void clean_up_mutexes() mysql_mutex_destroy(&LOCK_commit_ordered); mysql_mutex_destroy(&LOCK_slave_background); mysql_cond_destroy(&COND_slave_background); + mysql_cond_destroy(&COND_slave_list); #ifndef EMBEDDED_LIBRARY mysql_mutex_destroy(&LOCK_error_log); #endif @@ -4521,6 +4639,7 @@ static int init_thread_environment() mysql_mutex_init(key_LOCK_server_started, &LOCK_server_started, MY_MUTEX_INIT_FAST); mysql_cond_init(key_COND_server_started, &COND_server_started, NULL); + mysql_cond_init(key_COND_slave_list, &COND_slave_list, NULL); sp_cache_init(); #ifdef HAVE_EVENT_SCHEDULER Events::init_mutexes(); diff --git a/sql/mysqld.h b/sql/mysqld.h index 3d056fb95c7..bc307582807 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -382,7 +382,8 @@ extern PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, key_rpl_group_info_sleep_cond, key_TABLE_SHARE_cond, key_user_level_lock_cond, key_COND_start_thread, - key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache; + key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, + key_COND_slave_list; extern PSI_cond_key key_RELAYLOG_COND_relay_log_updated, key_RELAYLOG_COND_bin_log_updated, key_COND_wakeup_ready, key_COND_wait_commit; @@ -640,6 +641,7 @@ extern mysql_rwlock_t LOCK_grant, LOCK_sys_init_connect, LOCK_sys_init_slave; extern mysql_rwlock_t LOCK_ssl_refresh; extern mysql_prlock_t LOCK_system_variables_hash; extern mysql_cond_t COND_thread_count, COND_start_thread; +extern mysql_cond_t COND_slave_list; extern mysql_cond_t COND_manager; extern mysql_cond_t COND_slave_background; extern Atomic_counter<uint32_t> thread_count; diff --git a/sql/repl_failsafe.cc b/sql/repl_failsafe.cc index 68c7158e9e5..40d46c1f133 100644 --- a/sql/repl_failsafe.cc +++ b/sql/repl_failsafe.cc @@ -100,6 +100,9 @@ void unregister_slave(THD* thd, bool only_mine, bool need_mutex) (!only_mine || old_si->thd == thd)) my_hash_delete(&slave_list, (uchar*)old_si); + + if (slave_list.records == 0) + mysql_cond_broadcast(&COND_slave_list); if (need_mutex) mysql_mutex_unlock(&LOCK_slave_list); } @@ -152,6 +155,7 @@ int register_slave(THD* thd, uchar* packet, size_t packet_length) unregister_slave(thd,0,0); res= my_hash_insert(&slave_list, (uchar*) si); mysql_mutex_unlock(&LOCK_slave_list); + thd->rpl_dump_thread= true; return res; err: diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 81db3c286ad..5c25fa12952 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -602,7 +602,7 @@ extern "C" void thd_kill_timeout(THD* thd) THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) :Statement(&main_lex, &main_mem_root, STMT_CONVENTIONAL_EXECUTION, /* statement id */ 0), - rli_fake(0), rgi_fake(0), rgi_slave(NULL), + rli_fake(0), rgi_fake(0), rgi_slave(NULL), rpl_dump_thread(false), protocol_text(this), protocol_binary(this), m_current_stage_key(0), in_sub_stmt(0), log_all_errors(0), diff --git a/sql/sql_class.h b/sql/sql_class.h index 402a114aadd..4d5b26c696f 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -2206,6 +2206,7 @@ class THD: public THD_count, /* this must be first */ rpl_io_thread_info *rpl_io_info; rpl_sql_thread_info *rpl_sql_info; } system_thread_info; + bool rpl_dump_thread; MDL_ticket *mdl_backup_ticket, *mdl_backup_lock; void reset_for_next_command(bool do_clear_errors= 1); diff --git a/sql/sql_lex.h b/sql/sql_lex.h index c22f95978a0..5e73633a1b9 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -3092,6 +3092,7 @@ struct LEX: public Query_tables_list /* The following is used by KILL */ killed_state kill_signal; killed_type kill_type; + bool is_slaves_wait_shutdown; /* This variable is used in post-parse stage to declare that sum-functions, or functions which have sense only if GROUP BY is present, are allowed. diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 1335461506d..a6290a69cdb 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2145,6 +2145,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, */ enum mysql_enum_shutdown_level level; level= (enum mysql_enum_shutdown_level) (uchar) packet[0]; + thd->lex->is_slaves_wait_shutdown= false; // "deferred" cleanup if (level == SHUTDOWN_DEFAULT) level= SHUTDOWN_WAIT_ALL_BUFFERS; // soon default will be configurable else if (level != SHUTDOWN_WAIT_ALL_BUFFERS) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index fc08399bb88..2a48753e4ae 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2126,8 +2126,11 @@ static int init_binlog_sender(binlog_send_info *info, // set current pos too linfo->pos= *pos; + // dump thread linfo set is protected to avoid any slow shutdown race + mysql_mutex_lock(&LOCK_slave_list); // note: publish that we use file, before we open it thd->current_linfo= linfo; + mysql_mutex_unlock(&LOCK_slave_list); if (check_start_offset(info, linfo->log_file_name, *pos)) return 1; @@ -2393,7 +2396,8 @@ static int wait_new_events(binlog_send_info *info, /* in */ &stage_master_has_sent_all_binlog_to_slave, &old_stage); - while (!should_stop(info)) + while (!should_stop(info) && + mysql_bin_log.slaves_wait_shutdown != MYSQL_BIN_LOG::SHDN_DOIT) { *end_pos_ptr= mysql_bin_log.get_binlog_end_pos(binlog_end_pos_filename); if (strcmp(linfo->log_file_name, binlog_end_pos_filename) != 0) @@ -2518,7 +2522,8 @@ static my_off_t get_binlog_end_pos(binlog_send_info *info, * check if we should wait for more data */ if ((info->flags & BINLOG_DUMP_NON_BLOCK) || - (info->thd->variables.server_id == 0)) + (info->thd->variables.server_id == 0) || + mysql_bin_log.slaves_wait_shutdown == MYSQL_BIN_LOG::SHDN_DOIT) { info->should_stop= true; return 0; @@ -2745,6 +2750,14 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, info->error= ER_UNKNOWN_ERROR; goto err; } + DBUG_EXECUTE_IF("simulate_delay_at_shutdown", + { + const char act[]= + "now " + "WAIT_FOR greetings_from_kill_mysql"; + DBUG_ASSERT(!debug_sync_set_action(thd, + STRING_WITH_LEN(act))); + };); /* heartbeat_period from @master_heartbeat_period user variable diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 81ece13e1c5..9954937b676 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -14698,6 +14698,15 @@ kill_expr: shutdown: SHUTDOWN { Lex->sql_command= SQLCOM_SHUTDOWN; } + shutdown_option {} + ; + +shutdown_option: + /* Empty */ { Lex->is_slaves_wait_shutdown= false; } + | WAIT_SYM FOR_SYM SLAVES + { + Lex->is_slaves_wait_shutdown= true; + } ; /* change database */ diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index aa622b61771..1a380ffdbf4 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -14750,8 +14750,16 @@ kill_expr: shutdown: SHUTDOWN { Lex->sql_command= SQLCOM_SHUTDOWN; } + shutdown_option {} ; +shutdown_option: + /* Empty */ { Lex->is_slaves_wait_shutdown= false; } + | WAIT_SYM FOR_SYM SLAVES + { + Lex->is_slaves_wait_shutdown= true; + } + ; /* change database */ use: