[Commits] 3b0ac8fa013: MDEV-17062: Test failure on galera.MW-336

revision-id: 3b0ac8fa013d0a56bce736fb4283a35409394d71 (mariadb-10.1.35-35-g3b0ac8fa013) parent(s): 6695fcead3fb081109f04897c7a865d6c1bee275 author: Jan Lindström committer: Jan Lindström timestamp: 2018-09-06 10:29:27 +0300 message: MDEV-17062: Test failure on galera.MW-336 MDEV-17058: Test failure on wsrep.variables MDEV-17060: Test failure on galera.galera_var_slave_threads Fix incorrect calculation of increased applier (slave) threads. Note that increase change takes effect "immediately" but we should use proper wait condition to wait it. Reducing the number of slave threads is not immediate as thread will only exit after a replication event. --- mysql-test/suite/galera/r/MW-336.result | 42 ++++---- .../suite/galera/r/galera_var_slave_threads.result | 64 ------------ mysql-test/suite/galera/t/MW-336.test | 108 ++++++++++++++++----- .../suite/galera/t/galera_var_slave_threads.test | 13 +++ .../suite/wsrep/include/check_galera_version.inc | 20 ++-- mysql-test/suite/wsrep/r/variables.result | 5 + mysql-test/suite/wsrep/t/variables.test | 14 ++- sql/wsrep_var.cc | 4 +- 8 files changed, 145 insertions(+), 125 deletions(-) diff --git a/mysql-test/suite/galera/r/MW-336.result b/mysql-test/suite/galera/r/MW-336.result index 34874198c6f..81e8eae0eb3 100644 --- a/mysql-test/suite/galera/r/MW-336.result +++ b/mysql-test/suite/galera/r/MW-336.result @@ -1,33 +1,31 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; +INSERT INTO t1 values(0); SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 1; -INSERT INTO t1 VALUES (1); +# Wait 10 slave threads to start 1 +# Generate 12 replication events +SELECT COUNT(*) FROM t1; +COUNT(*) +13 +# Wait 9 slave threads to exit 1 SET GLOBAL wsrep_slave_threads = 10; +# Wait 10 slave threads to start 2 SET GLOBAL wsrep_slave_threads = 20; +# Wait 20 slave threads to start 3 SET GLOBAL wsrep_slave_threads = 1; -INSERT INTO t1 VALUES (1); -INSERT INTO t1 VALUES (2); -INSERT INTO t1 VALUES (3); -INSERT INTO t1 VALUES (4); -INSERT INTO t1 VALUES (5); -INSERT INTO t1 VALUES (6); -INSERT INTO t1 VALUES (7); -INSERT INTO t1 VALUES (8); -INSERT INTO t1 VALUES (9); +# Generate 40 replication events +SELECT COUNT(*) FROM t1; +COUNT(*) +53 +# Wait 10 slave threads to exit 3 SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 0; Warnings: Warning 1292 Truncated incorrect wsrep_slave_threads value: '0' -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); -INSERT INTO t1 VALUES (20); -SET GLOBAL wsrep_slave_threads = 1; +# Wait 10 slave threads to start 3 +# Generate 12 replication events +SELECT COUNT(*) FROM t1; +COUNT(*) +65 +# Wait 10 slave threads to exit 4 DROP TABLE t1; diff --git a/mysql-test/suite/galera/r/galera_var_slave_threads.result b/mysql-test/suite/galera/r/galera_var_slave_threads.result index 70f3b8ee257..c7c6af2098f 100644 --- a/mysql-test/suite/galera/r/galera_var_slave_threads.result +++ b/mysql-test/suite/galera/r/galera_var_slave_threads.result @@ -26,70 +26,6 @@ SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system use COUNT(*) = 1 1 SET GLOBAL wsrep_slave_threads = 1; -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); -INSERT INTO t2 VALUES (DEFAULT); SELECT COUNT(*) = 64 FROM t2; COUNT(*) = 64 1 diff --git a/mysql-test/suite/galera/t/MW-336.test b/mysql-test/suite/galera/t/MW-336.test index 9572489ebe9..749ffe671be 100644 --- a/mysql-test/suite/galera/t/MW-336.test +++ b/mysql-test/suite/galera/t/MW-336.test @@ -6,62 +6,118 @@ --source include/have_innodb.inc CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; +INSERT INTO t1 values(0); --connection node_1 + SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 1; ---let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; + +--echo # Wait 10 slave threads to start 1 +--let $wait_timeout=600 +--let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc --connection node_2 -INSERT INTO t1 VALUES (1); +# Wait until inserts are replicated +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc +--echo # Generate 12 replication events +--disable_query_log +--disable_result_log +--let $count = 12 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} +--enable_result_log +--enable_query_log --connection node_1 +# Wait until inserts are replicated +--let $wait_condition = SELECT COUNT(*) = 13 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) FROM t1; + +--echo # Wait 9 slave threads to exit 1 +# Wait until appliers exit +--let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +--source include/wait_condition.inc + SET GLOBAL wsrep_slave_threads = 10; + +--echo # Wait 10 slave threads to start 2 --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; --source include/wait_condition.inc SET GLOBAL wsrep_slave_threads = 20; + +--echo # Wait 20 slave threads to start 3 --let $wait_condition = SELECT COUNT(*) = 21 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; --source include/wait_condition.inc SET GLOBAL wsrep_slave_threads = 1; --connection node_2 -INSERT INTO t1 VALUES (1); -INSERT INTO t1 VALUES (2); -INSERT INTO t1 VALUES (3); -INSERT INTO t1 VALUES (4); -INSERT INTO t1 VALUES (5); -INSERT INTO t1 VALUES (6); -INSERT INTO t1 VALUES (7); -INSERT INTO t1 VALUES (8); -INSERT INTO t1 VALUES (9); - +--echo # Generate 40 replication events +--disable_query_log +--disable_result_log +--let $count = 40 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} +--enable_query_log +--enable_result_log --connection node_1 ---let $wait_condition = SELECT COUNT(*) = 12 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; + +# Wait until inserts are replicated +--let $wait_condition = SELECT COUNT(*) = 53 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) FROM t1; + +--echo # Wait 10 slave threads to exit 3 +# Wait until appliers exit +--let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 0; +--echo # Wait 10 slave threads to start 3 +--let $wait_timeout=600 +--let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +--source include/wait_condition.inc + --connection node_2 -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); -INSERT INTO t1 VALUES (20); +--echo # Generate 12 replication events +--disable_query_log +--disable_result_log +--let $count = 12 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} +--enable_result_log +--enable_query_log --connection node_1 ---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; +# Wait until inserts are replicated +--let $wait_condition = SELECT COUNT(*) = 65 FROM t1; --source include/wait_condition.inc -SET GLOBAL wsrep_slave_threads = 1; +SELECT COUNT(*) FROM t1; + +--echo # Wait 10 slave threads to exit 4 +# Wait until appliers exit +--let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +--source include/wait_condition.inc + +--connection node_1 DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_var_slave_threads.test b/mysql-test/suite/galera/t/galera_var_slave_threads.test index 0afda948ed5..1cee845b6ab 100644 --- a/mysql-test/suite/galera/t/galera_var_slave_threads.test +++ b/mysql-test/suite/galera/t/galera_var_slave_threads.test @@ -34,7 +34,12 @@ SET GLOBAL wsrep_slave_threads = 64; INSERT INTO t1 VALUES (1); --connection node_2 +--let $wait_timeout=600 +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + SELECT COUNT(*) = 1 FROM t1; + --let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; --source include/wait_condition.inc SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%'; @@ -47,6 +52,8 @@ SET GLOBAL wsrep_slave_threads = 1; --connection node_1 +--disable_result_log +--disable_query_log # Generate 64 replication events --let $count = 64 while ($count) @@ -54,9 +61,15 @@ while ($count) INSERT INTO t2 VALUES (DEFAULT); --dec $count } +--enable_query_log +--enable_result_log --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 64 FROM t2; +--source include/wait_condition.inc + SELECT COUNT(*) = 64 FROM t2; + --let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; --source include/wait_condition.inc SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%'; diff --git a/mysql-test/suite/wsrep/include/check_galera_version.inc b/mysql-test/suite/wsrep/include/check_galera_version.inc index e495da8f1ee..fd691161a54 100644 --- a/mysql-test/suite/wsrep/include/check_galera_version.inc +++ b/mysql-test/suite/wsrep/include/check_galera_version.inc @@ -17,23 +17,27 @@ eval SET @GALERA_VERSION='$galera_version'; SELECT CAST(REGEXP_REPLACE(@GALERA_VERSION,'^(\\d+)\\.(\\d+).*','\\1') AS UNSIGNED) INTO @GALERA_MAJOR_VERSION; SELECT CAST(REGEXP_REPLACE(@GALERA_VERSION,'^(\\d+)\\.(\\d+).*','\\2') AS UNSIGNED) INTO @GALERA_MINOR_VERSION; +SELECT CAST(REGEXP_REPLACE(@GALERA_VERSION,'^(\\d+)\\.(\\d+)\\.(\\d+).*','\\3') AS UNSIGNED) INTO @GALERA_RELEASE_VERSION; # Actual SELECT VARIABLE_VALUE INTO @ACTUAL_GALERA_VERSION FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME LIKE 'wsrep_provider_version'; - -SELECT CAST(REGEXP_REPLACE(@ACTUAL_GALERA_VERSION,'^(\\d+)\\.(\\d+).*','\\1') AS UNSIGNED) INTO @ACTUAL_GALERA_MAJOR_VERSION; -SELECT CAST(REGEXP_REPLACE(@ACTUAL_GALERA_VERSION,'^(\\d+)\\.(\\d+).*','\\2') AS UNSIGNED) INTO @ACTUAL_GALERA_MINOR_VERSION; +SELECT CAST(REGEXP_REPLACE(@ACTUAL_GALERA_VERSION,'^(\\d+)\\.(\\d+).*','\\1') AS UNSIGNED) INTO @ACTUAL_GALERA_MINOR_VERSION; +SELECT CAST(REGEXP_REPLACE(@ACTUAL_GALERA_VERSION,'^(\\d+)\\.(\\d+).*','\\2') AS UNSIGNED) INTO @ACTUAL_GALERA_RELEASE_VERSION; # For testing -#SELECT @GALERA_MAJOR_VERSION, @GALERA_MINOR_VERSION; +#SELECT @GALERA_MAJOR_VERSION; +#SELECT @GALERA_MINOR_VERSION; +#SELECT @GALERA_RELEASE_VERSION; #SELECT @ACTUAL_GALERA_VERSION; -#SELECT @ACTUAL_GALERA_MAJOR_VERSION, @ACTUAL_GALERA_MINOR_VERSION; +#SELECT @ACTUAL_GALERA_MINOR_VERSION; +#SELECT @ACTUAL_GALERA_RELEASE_VERSION; -if (!`SELECT (@ACTUAL_GALERA_MAJOR_VERSION > @GALERA_MAJOR_VERSION) OR - (@ACTUAL_GALERA_MAJOR_VERSION = @GALERA_MAJOR_VERSION AND @ACTUAL_GALERA_MINOR_VERSION >= @GALERA_MINOR_VERSION) +if (!`SELECT (@ACTUAL_GALERA_MINOR_VERSION > @GALERA_MINOR_VERSION) OR + (@ACTUAL_GALERA_MINOR_VERSION = @GALERA_MINOR_VERSION AND + @ACTUAL_GALERA_RELEASE_VERSION >= @GALERA_RELEASE_VERSION) `) { - skip Test requires Galera library version $galera_version; + skip Test requires Galera library version >= $galera_version; } --enable_query_log diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result index 4ecc1ca3b79..7489aa0d914 100644 --- a/mysql-test/suite/wsrep/r/variables.result +++ b/mysql-test/suite/wsrep/r/variables.result @@ -19,6 +19,7 @@ SET GLOBAL wsrep_provider=none; # variables when using "_" # CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*"); +SET GLOBAL wsrep_provider= '/usr/lib/galera/libgalera_smm.so'; SHOW GLOBAL STATUS LIKE 'wsrep%'; Variable_name Value wsrep_apply_oooe # @@ -59,6 +60,8 @@ wsrep_local_send_queue_min # wsrep_local_state # wsrep_local_state_comment # wsrep_local_state_uuid # +wsrep_open_connections # +wsrep_open_transactions # wsrep_protocol_version # wsrep_provider_name # wsrep_provider_vendor # @@ -114,6 +117,8 @@ wsrep_local_send_queue_min # wsrep_local_state # wsrep_local_state_comment # wsrep_local_state_uuid # +wsrep_open_connections # +wsrep_open_transactions # wsrep_protocol_version # wsrep_provider_name # wsrep_provider_vendor # diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 1315f090d5c..ca71baaba02 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -27,11 +27,11 @@ SET GLOBAL wsrep_provider=none; CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*"); ---disable_query_log +--disable_result_log eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER'; --let $galera_version=25.3.17 source include/check_galera_version.inc; ---enable_query_log +--enable_result_log --replace_column 2 # SHOW GLOBAL STATUS LIKE 'wsrep%'; @@ -101,8 +101,11 @@ SHOW STATUS LIKE 'wsrep_thread_count'; --echo # Setting wsrep_cluster_address triggers the creation of --echo # applier/rollbacker threads. SET GLOBAL wsrep_cluster_address= 'gcomm://'; + --echo # Wait for applier threads to get created. -sleep 3; +--let $wait_timeout=600 +--let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +--source include/wait_condition.inc --replace_regex /.*libgalera_smm.*/libgalera_smm.so/ SELECT @@global.wsrep_provider; @@ -113,8 +116,11 @@ SHOW STATUS LIKE 'wsrep_thread_count'; SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads; SET GLOBAL wsrep_slave_threads= 10; + --echo # Wait for applier threads to get created. -sleep 3; +--let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +--source include/wait_condition.inc + SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 3826ebed14c..eea1da1d46c 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -564,7 +564,9 @@ void wsrep_node_address_init (const char* value) static void wsrep_slave_count_change_update () { - wsrep_slave_count_change += (wsrep_slave_threads - wsrep_prev_slave_threads); + wsrep_slave_count_change = (wsrep_slave_threads - wsrep_prev_slave_threads); + WSREP_DEBUG("Change on slave threads: New %lu old %lu difference %lu", + wsrep_slave_threads, wsrep_prev_slave_threads, wsrep_slave_count_change); wsrep_prev_slave_threads = wsrep_slave_threads; }