- commits - lists.mariadb.org

[Commits] 068ea0a7cb5: MDEV-21243: Join buffer: condition is checked in wrong place for range access
by psergey 12 Jan '20

12 Jan '20

revision-id: 068ea0a7cb5645fd28e0df0ad3a17f5a5e2392f7 (mariadb-10.3.21-1-g068ea0a7cb5) parent(s): e5e5877740f248de848219ee3a1d2881cd5c5b82 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-12 15:47:23 +0300 message: MDEV-21243: Join buffer: condition is checked in wrong place for range access In this scenario: - There is a possible range access for table T - And there is a ref access on the same index which uses fewer key parts - The join optimizer picks the ref access (because it is cheaper) - make_join_select applies this heuristic to switch to range: /* Range uses longer key; Use this instead of ref on key */ Join buffer will be used without having called JOIN_TAB::make_scan_filter(). This means, conditions that should be checked when reading table T will be checked after T is joined with the contents of the join buffer, instead. Fixed this by adding a make_scan_filter() check. (updated patch after backport to 10.3) (Fix testcase on Windows) --- mysql-test/main/join_cache.result | 72 +++++++++++++++++++++++++++++++++++++++ mysql-test/main/join_cache.test | 40 ++++++++++++++++++++++ sql/sql_select.cc | 10 ++++++ 3 files changed, 122 insertions(+) diff --git a/mysql-test/main/join_cache.result b/mysql-test/main/join_cache.result index bab32395413..3d1d91df997 100644 --- a/mysql-test/main/join_cache.result +++ b/mysql-test/main/join_cache.result @@ -6056,4 +6056,76 @@ select f2 from t2,t1 where f2 = 0; f2 drop table t1, t2; set join_buffer_size=@save_join_buffer_size; +# +# MDEV-21243: Join buffer: condition is checked in wrong place for range access +# +create table t1(a int primary key); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t2 (a int); +insert into t2 select A.a + 10*B.a from t1 A, t1 B; +create table t3 ( +kp1 int, +kp2 int, +col1 int, +col2 int, +key (kp1, kp2) +); +insert into t3 +select +A.a, +B.a, +A.a + 100*B.a, +A.a + 100*B.a +from +t2 A, t2 B; +analyze table t3; +Table Op Msg_type Msg_text +test.t3 analyze status Table is already up to date +# The following must have "B.col1 + 1 < 33333" attached to table B +# and not to the block-nl-join node: +explain format=json +select * +from t1 a, t3 b +where +b.kp1=a.a and +b.kp1 <= 10 and +b.kp2 <= 10 and +b.col1 +1 < 33333; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "a", + "access_type": "index", + "possible_keys": ["PRIMARY"], + "key": "PRIMARY", + "key_length": "4", + "used_key_parts": ["a"], + "rows": 10, + "filtered": 100, + "attached_condition": "a.a <= 10", + "using_index": true + }, + "block-nl-join": { + "table": { + "table_name": "b", + "access_type": "range", + "possible_keys": ["kp1"], + "key": "kp1", + "key_length": "10", + "used_key_parts": ["kp1", "kp2"], + "rows": 836, + "filtered": 11.962, + "index_condition": "b.kp2 <= 10", + "attached_condition": "b.kp2 <= 10 and b.col1 + 1 < 33333" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "b.kp1 = a.a" + } + } +} +drop table t1,t2,t3; set @@optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/main/join_cache.test b/mysql-test/main/join_cache.test index 9576d598125..91339c2cb21 100644 --- a/mysql-test/main/join_cache.test +++ b/mysql-test/main/join_cache.test @@ -4014,5 +4014,45 @@ select f2 from t2,t1 where f2 = 0; drop table t1, t2; set join_buffer_size=@save_join_buffer_size; + +--echo # +--echo # MDEV-21243: Join buffer: condition is checked in wrong place for range access +--echo # +create table t1(a int primary key); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t2 (a int); +insert into t2 select A.a + 10*B.a from t1 A, t1 B; + +create table t3 ( + kp1 int, + kp2 int, + col1 int, + col2 int, + key (kp1, kp2) +); + +insert into t3 +select + A.a, + B.a, + A.a + 100*B.a, + A.a + 100*B.a +from + t2 A, t2 B; +analyze table t3; + +--echo # The following must have "B.col1 + 1 < 33333" attached to table B +--echo # and not to the block-nl-join node: +explain format=json +select * +from t1 a, t3 b +where + b.kp1=a.a and + b.kp1 <= 10 and + b.kp2 <= 10 and + b.col1 +1 < 33333; + +drop table t1,t2,t3; + # The following command must be the last one in the file set @@optimizer_switch=@save_optimizer_switch; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 275c46141cd..fbce8708c79 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -10960,6 +10960,13 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) if (i != join->const_tables && tab->use_quick != 2 && !tab->first_inner) { /* Read with cache */ + /* + TODO: the execution also gets here when we will not be using + join buffer. Review these cases and perhaps, remove this call. + (The final decision whether to use join buffer is made in + check_join_cache_usage, so we should only call make_scan_filter() + there, too). + */ if (tab->make_scan_filter()) DBUG_RETURN(1); } @@ -11922,6 +11929,9 @@ uint check_join_cache_usage(JOIN_TAB *tab, if ((tab->cache= new (root) JOIN_CACHE_BNL(join, tab, prev_cache))) { tab->icp_other_tables_ok= FALSE; + /* If make_join_select() hasn't called make_scan_filter(), do it now */ + if (!tab->cache_select && tab->make_scan_filter()) + goto no_join_cache; return (2 - MY_TEST(!prev_cache)); } goto no_join_cache;

1 0

[Commits] bf55fb035be: TEst
by Sergei Petrunia 11 Jan '20

11 Jan '20

revision-id: bf55fb035be1efe84c2d8419948f42c0d43ba7d6 (fb-prod201903-205-gbf55fb035be) parent(s): 8c227943326d1e420215ac983c9e503f6fc86e80 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-11 01:13:52 +0200 message: TEst --- .../suite/rocksdb_sys_vars/r/all_vars.result | 2 - .../r/rocksdb_mrr_batch_size_basic.result | 93 ++++++++++++++++++++++ .../t/rocksdb_mrr_batch_size_basic.test | 17 ++++ 3 files changed, 110 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/rocksdb_sys_vars/r/all_vars.result b/mysql-test/suite/rocksdb_sys_vars/r/all_vars.result index bb96a1cb1c8..159d6a983c8 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/all_vars.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/all_vars.result @@ -9,7 +9,5 @@ There should be *no* long test name listed below: select variable_name as `There should be *no* variables listed below:` from t2 left join t1 on variable_name=test_name where test_name is null ORDER BY variable_name; There should be *no* variables listed below: -ROCKSDB_MRR_BATCH_SIZE -ROCKSDB_MRR_BATCH_SIZE drop table t1; drop table t2; diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_mrr_batch_size_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_mrr_batch_size_basic.result new file mode 100644 index 00000000000..5b7cc20d6fb --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_mrr_batch_size_basic.result @@ -0,0 +1,93 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(100); +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +SET @start_global_value = @@global.ROCKSDB_MRR_BATCH_SIZE; +SELECT @start_global_value; +@start_global_value +100 +SET @start_session_value = @@session.ROCKSDB_MRR_BATCH_SIZE; +SELECT @start_session_value; +@start_session_value +100 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_MRR_BATCH_SIZE to 100" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = 100; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +100 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = DEFAULT; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +100 +"Trying to set variable @@global.ROCKSDB_MRR_BATCH_SIZE to 1" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = 1; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = DEFAULT; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +100 +"Trying to set variable @@global.ROCKSDB_MRR_BATCH_SIZE to 0" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = 0; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = DEFAULT; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +100 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_MRR_BATCH_SIZE to 100" +SET @@session.ROCKSDB_MRR_BATCH_SIZE = 100; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +100 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MRR_BATCH_SIZE = DEFAULT; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +100 +"Trying to set variable @@session.ROCKSDB_MRR_BATCH_SIZE to 1" +SET @@session.ROCKSDB_MRR_BATCH_SIZE = 1; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MRR_BATCH_SIZE = DEFAULT; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +100 +"Trying to set variable @@session.ROCKSDB_MRR_BATCH_SIZE to 0" +SET @@session.ROCKSDB_MRR_BATCH_SIZE = 0; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MRR_BATCH_SIZE = DEFAULT; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +100 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_MRR_BATCH_SIZE to 'aaa'" +SET @@global.ROCKSDB_MRR_BATCH_SIZE = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +100 +SET @@global.ROCKSDB_MRR_BATCH_SIZE = @start_global_value; +SELECT @@global.ROCKSDB_MRR_BATCH_SIZE; +@@global.ROCKSDB_MRR_BATCH_SIZE +100 +SET @@session.ROCKSDB_MRR_BATCH_SIZE = @start_session_value; +SELECT @@session.ROCKSDB_MRR_BATCH_SIZE; +@@session.ROCKSDB_MRR_BATCH_SIZE +100 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_mrr_batch_size_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_mrr_batch_size_basic.test new file mode 100644 index 00000000000..a45a86d5fbe --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_mrr_batch_size_basic.test @@ -0,0 +1,17 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(100); +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); + +--let $sys_var=ROCKSDB_MRR_BATCH_SIZE +--let $read_only=0 +--let $session=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values;

1 0

[Commits] 4e2c026a4cb: MDEV-21377: Eq_ref access not picked by query with optimizer_use_condition_selectivity > 1
by Varun 10 Jan '20

10 Jan '20

revision-id: 4e2c026a4cbde7ea6b2e3e1be87333c48a719c08 (mariadb-10.4.11-27-g4e2c026a4cb) parent(s): 983163209d026bfd979b4298053fcbdb373efa9d author: Varun Gupta committer: Varun Gupta timestamp: 2020-01-10 17:27:28 +0530 message: MDEV-21377: Eq_ref access not picked by query with optimizer_use_condition_selectivity > 1 The issue here is the estimate of the number of distinct ref access made by the optimizer are greater than the records in the join prefix. This is incorrect, we need to make sure that the number of distinct ref accesses are less than the records in the join prefix --- mysql-test/main/selectivity.result | 56 +++++++++++++++++++++++++++++++ mysql-test/main/selectivity.test | 38 +++++++++++++++++++++ mysql-test/main/selectivity_innodb.result | 56 +++++++++++++++++++++++++++++++ sql/sql_select.cc | 3 ++ 4 files changed, 153 insertions(+) diff --git a/mysql-test/main/selectivity.result b/mysql-test/main/selectivity.result index 4366ef6a564..95069c74391 100644 --- a/mysql-test/main/selectivity.result +++ b/mysql-test/main/selectivity.result @@ -1867,4 +1867,60 @@ set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; # End of 10.1 tests +# +# MDEV-21377: Eq_ref access not picked by query with +# optimizer_use_condition_selectivity > 1 +# +create table t1(a int, b int,c int, primary key(a), key(b), key(c)); +insert into t1 select seq, seq, seq from seq_1_to_100; +create table t2(a int, b int,c int, primary key(a), key(b), key(c)); +insert into t2 select seq, seq, seq from seq_1_to_1000; +create table t3(a int, b int, primary key(a)); +insert into t3 select seq, seq from seq_1_to_100; +analyze table t1,t2,t3; +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_optimizer_switch=@@optimizer_switch; +set optimizer_switch='rowid_filter=off'; +set optimizer_use_condition_selectivity=1; +explain SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range PRIMARY,b,c b 5 NULL 5 Using index condition; Using where +1 SIMPLE t2 ref b,c c 5 test.t1.c 1 Using where +1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 +SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +a b c a b c a b +1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 +5 5 5 5 5 5 5 5 +6 6 6 6 6 6 6 6 +7 7 7 7 7 7 7 7 +8 8 8 8 8 8 8 8 +9 9 9 9 9 9 9 9 +set optimizer_use_condition_selectivity=4; +explain SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range PRIMARY,b,c b 5 NULL 5 Using index condition; Using where +1 SIMPLE t2 ref b,c c 5 test.t1.c 1 Using where +1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 +SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +a b c a b c a b +1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 +5 5 5 5 5 5 5 5 +6 6 6 6 6 6 6 6 +7 7 7 7 7 7 7 7 +8 8 8 8 8 8 8 8 +9 9 9 9 9 9 9 9 +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set optimizer_switch=@save_optimizer_switch; +drop table t1,t2,t3; +# End of 10.4 tests set @@global.histogram_size=@save_histogram_size; diff --git a/mysql-test/main/selectivity.test b/mysql-test/main/selectivity.test index d0158fb717e..6ed3ae67d83 100644 --- a/mysql-test/main/selectivity.test +++ b/mysql-test/main/selectivity.test @@ -1269,6 +1269,44 @@ drop table t1,t2; --echo # End of 10.1 tests +--echo # +--echo # MDEV-21377: Eq_ref access not picked by query with +--echo # optimizer_use_condition_selectivity > 1 +--echo # + +create table t1(a int, b int,c int, primary key(a), key(b), key(c)); +insert into t1 select seq, seq, seq from seq_1_to_100; + +create table t2(a int, b int,c int, primary key(a), key(b), key(c)); +insert into t2 select seq, seq, seq from seq_1_to_1000; + +create table t3(a int, b int, primary key(a)); +insert into t3 select seq, seq from seq_1_to_100; +--disable_result_log +analyze table t1,t2,t3; +--enable_result_log + +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_optimizer_switch=@@optimizer_switch; +set optimizer_switch='rowid_filter=off'; +let $q= SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and + t2.b < 10 AND t1.b < 10; + +set optimizer_use_condition_selectivity=1; +eval explain $q; +eval $q; + +set optimizer_use_condition_selectivity=4; +eval explain $q; +eval $q; + +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set optimizer_switch=@save_optimizer_switch; + +drop table t1,t2,t3; + +--echo # End of 10.4 tests + # # Clean up # diff --git a/mysql-test/main/selectivity_innodb.result b/mysql-test/main/selectivity_innodb.result index 062b2da7b4f..d48848a92bf 100644 --- a/mysql-test/main/selectivity_innodb.result +++ b/mysql-test/main/selectivity_innodb.result @@ -1877,6 +1877,62 @@ set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; # End of 10.1 tests +# +# MDEV-21377: Eq_ref access not picked by query with +# optimizer_use_condition_selectivity > 1 +# +create table t1(a int, b int,c int, primary key(a), key(b), key(c)); +insert into t1 select seq, seq, seq from seq_1_to_100; +create table t2(a int, b int,c int, primary key(a), key(b), key(c)); +insert into t2 select seq, seq, seq from seq_1_to_1000; +create table t3(a int, b int, primary key(a)); +insert into t3 select seq, seq from seq_1_to_100; +analyze table t1,t2,t3; +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_optimizer_switch=@@optimizer_switch; +set optimizer_switch='rowid_filter=off'; +set optimizer_use_condition_selectivity=1; +explain SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range PRIMARY,b,c b 5 NULL 9 Using index condition; Using where +1 SIMPLE t2 ref b,c c 5 test.t1.c 1 Using where +1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 +SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +a b c a b c a b +1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 +5 5 5 5 5 5 5 5 +6 6 6 6 6 6 6 6 +7 7 7 7 7 7 7 7 +8 8 8 8 8 8 8 8 +9 9 9 9 9 9 9 9 +set optimizer_use_condition_selectivity=4; +explain SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range PRIMARY,b,c b 5 NULL 9 Using index condition; Using where +1 SIMPLE t2 ref b,c c 5 test.t1.c 1 Using where +1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 +SELECT * FROM t1, t2, t3 WHERE t1.c= t2.c and t1.a = t3.a and +t2.b < 10 AND t1.b < 10; +a b c a b c a b +1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 +5 5 5 5 5 5 5 5 +6 6 6 6 6 6 6 6 +7 7 7 7 7 7 7 7 +8 8 8 8 8 8 8 8 +9 9 9 9 9 9 9 9 +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set optimizer_switch=@save_optimizer_switch; +drop table t1,t2,t3; +# End of 10.4 tests set @@global.histogram_size=@save_histogram_size; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index d9d9c229c2f..f5b08dcdd79 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -7370,6 +7370,7 @@ best_access_path(JOIN *join, double tmp2= prev_record_reads(join_positions, idx, (found_ref | keyuse->used_tables)); + tmp2= MY_MIN(tmp2, record_count); if (tmp2 < best_prev_record_reads) { best_part_found_ref= keyuse->used_tables & ~join->const_table_map; @@ -7411,6 +7412,7 @@ best_access_path(JOIN *join, but 1.0 would be probably safer */ tmp= prev_record_reads(join_positions, idx, found_ref); + MY_MIN(tmp, record_count); records= 1.0; type= JT_FT; trace_access_idx.add("access_type", join_type_str[type]) @@ -7440,6 +7442,7 @@ best_access_path(JOIN *join, trace_access_idx.add("access_type", join_type_str[type]) .add("index", keyinfo->name); tmp = prev_record_reads(join_positions, idx, found_ref); + tmp= MY_MIN(tmp, record_count); records=1.0; } else

1 0

[Commits] 62cde94a0c5: Fix compilation after 3339788
by Sergei Petrunia 09 Jan '20

09 Jan '20

revision-id: 62cde94a0c51d565a430a4c7e43d514fe14e64fc (fb-prod201903-199-g62cde94a0c5) parent(s): d8e7af55ca61ee23916eb693bf3c1f17cdab812f author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-09 14:53:17 +0100 message: Fix compilation after 3339788 Fix compilation after patch 3339788 Streaming compression Summary: Extend protocol to support streaming compression. ... --- include/mysql.h.pp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/mysql.h.pp b/include/mysql.h.pp index 17d40ebfea7..edf7a0990ea 100644 --- a/include/mysql.h.pp +++ b/include/mysql.h.pp @@ -77,7 +77,9 @@ typedef struct ssl_st SSL; enum mysql_compression_lib { MYSQL_COMPRESSION_NONE, MYSQL_COMPRESSION_ZLIB, - MYSQL_COMPRESSION_ZSTD + MYSQL_COMPRESSION_ZSTD, + MYSQL_COMPRESSION_ZSTD_STREAM, + MYSQL_COMPRESSION_LZ4F_STREAM }; typedef struct ZSTD_CCtx_s ZSTD_CCtx; typedef struct ZSTD_DCtx_s ZSTD_DCtx; @@ -102,6 +104,11 @@ typedef struct st_net { enum mysql_compression_lib comp_lib; ZSTD_CCtx *cctx; ZSTD_DCtx *dctx; + void *lz4f_cctx; + void *lz4f_dctx; + unsigned char* compress_buf; + unsigned long compress_buf_len; + my_bool reset_cctx; void *qsbr_context; unsigned char *unused; unsigned int last_errno;

1 0

[Commits] 1fc2a512dff: MDEV-21443: Donot report INCIDENT_EVENT in binary log for transactional changes which are safely rolledback
by sujatha 09 Jan '20

09 Jan '20

revision-id: 1fc2a512dffe9a41d58f4484c06128be100ccadd (mariadb-10.2.30-48-g1fc2a512dff) parent(s): 8317f77ccc490c94f2f43d2e6d4104c87f232563 author: Sujatha committer: Sujatha timestamp: 2020-01-09 14:19:04 +0530 message: MDEV-21443: Donot report INCIDENT_EVENT in binary log for transactional changes which are safely rolledback Problem: ======= Issue 1: Incident event is reported for transactions which are rolled back successfully. Issue 2: When the mariadb slave (error) stops at receiving the incident event there's no description of what led to it. Neither in the event nor in the master's error log. Analysis: ========= Incident events are logged when a multi statement transaction specific changes cannot be safely rolled back. In reported scenario a big multi statement transaction is in progress and it makes use of innodb transactional engine. Row based replication is being used. At the time of writing the annotate event for the multi statement transaction the binary log cache size exceeds the configured max_binlog_cache_size. Hence write to binary log fails. The write error handling code generates an incident event and writes to binary log. This stops the slave. The error code is propagated to the callers and transaction is rolled back. Ideally binlog write error handling code should restrict incident event reporting only for cases where clean rollback is not possible. At the time of reporting incident event the actual error message which lead to incident should also be written to server error log for debugging purpose. Fix: === Report incident event in cases where transactions cannot be safely rolledback. Write the error message into the error log. @sql/handler.cc Added new variable "stmt_modified_non_trans_table" to identify modification to non transactional tables as "thd->transaction.stmt.modified_non_trans_table" and "thd->transaction.all.modified_non_trans_table" will be set at the later stage of execution. Annotate/Table_map events are written at the starting stage of transaction execution. i.e In a multi statement transaction when the first statement execution is complete its changes are written to binary log. At this stage the Annotate/Table_map events are written. To set "stmt_modified_non_trans_table" iterate through the list of tables modified by the transaction and see if they are transactional or not. If table is non transactional and it has write mode lock then set this variable to true. Upon event write error check for the above flag and report incident event. @sql/log.cc Report incident event if "stmt_modified_non_trans_table" variable is set. In "write_incident" code print thd->error_message() to server error log. --- .../extra/binlog_tests/binlog_write_error.inc | 2 + .../extra/rpl_tests/rpl_binlog_max_cache_size.test | 3 + mysql-test/suite/binlog/r/binlog_bug23533.result | 1 + .../suite/binlog/r/binlog_write_error.result | 1 + mysql-test/suite/binlog/t/binlog_bug23533.test | 2 +- .../binlog_encryption/binlog_write_error.result | 1 + .../rpl_mixed_binlog_max_cache_size.result | 2 + mysql-test/suite/rpl/r/rpl_mdev-11092.result | 73 ++++++++++++- .../rpl/r/rpl_mixed_binlog_max_cache_size.result | 2 + .../rpl/r/rpl_stm_binlog_max_cache_size.result | 2 + mysql-test/suite/rpl/t/rpl_mdev-11092.test | 121 ++++++++++++++++++++- sql/handler.cc | 26 ++++- sql/log.cc | 54 +++++++-- sql/sql_class.h | 3 +- 14 files changed, 270 insertions(+), 23 deletions(-) diff --git a/mysql-test/extra/binlog_tests/binlog_write_error.inc b/mysql-test/extra/binlog_tests/binlog_write_error.inc index fa3ba087a7e..0ccde46ac48 100644 --- a/mysql-test/extra/binlog_tests/binlog_write_error.inc +++ b/mysql-test/extra/binlog_tests/binlog_write_error.inc @@ -38,6 +38,8 @@ DROP TRIGGER IF EXISTS tr2; DROP VIEW IF EXISTS v1, v2; enable_warnings; +call mtr.add_suppression("Write to binary log failed: Error writing file"); + --echo # --echo # Test injecting binlog write error when executing queries --echo # diff --git a/mysql-test/extra/rpl_tests/rpl_binlog_max_cache_size.test b/mysql-test/extra/rpl_tests/rpl_binlog_max_cache_size.test index 0f46b00f683..1d04bbcc215 100644 --- a/mysql-test/extra/rpl_tests/rpl_binlog_max_cache_size.test +++ b/mysql-test/extra/rpl_tests/rpl_binlog_max_cache_size.test @@ -22,6 +22,9 @@ # ######################################################################################## call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +call mtr.add_suppression("Write to binary log failed: Multi-row statements required more than .max_binlog_stmt_cache_size"); +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); + let $old_max_binlog_cache_size= query_get_value(SHOW VARIABLES LIKE "max_binlog_cache_size", Value, 1); let $old_binlog_cache_size= query_get_value(SHOW VARIABLES LIKE "binlog_cache_size", Value, 1); diff --git a/mysql-test/suite/binlog/r/binlog_bug23533.result b/mysql-test/suite/binlog/r/binlog_bug23533.result index cc9799506c3..4f978754b95 100644 --- a/mysql-test/suite/binlog/r/binlog_bug23533.result +++ b/mysql-test/suite/binlog/r/binlog_bug23533.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); SET AUTOCOMMIT=0; CREATE TABLE t1 (a INT NOT NULL AUTO_INCREMENT, b TEXT, PRIMARY KEY(a)) ENGINE=InnoDB; SELECT COUNT(*) FROM t1; diff --git a/mysql-test/suite/binlog/r/binlog_write_error.result b/mysql-test/suite/binlog/r/binlog_write_error.result index 2606a9f40b3..46cc346bb70 100644 --- a/mysql-test/suite/binlog/r/binlog_write_error.result +++ b/mysql-test/suite/binlog/r/binlog_write_error.result @@ -9,6 +9,7 @@ DROP PROCEDURE IF EXISTS p2; DROP TRIGGER IF EXISTS tr1; DROP TRIGGER IF EXISTS tr2; DROP VIEW IF EXISTS v1, v2; +call mtr.add_suppression("Write to binary log failed: Error writing file"); # # Test injecting binlog write error when executing queries # diff --git a/mysql-test/suite/binlog/t/binlog_bug23533.test b/mysql-test/suite/binlog/t/binlog_bug23533.test index ca610e399e4..5f0f1d894f3 100644 --- a/mysql-test/suite/binlog/t/binlog_bug23533.test +++ b/mysql-test/suite/binlog/t/binlog_bug23533.test @@ -6,7 +6,7 @@ --source include/have_innodb.inc --source include/have_log_bin.inc --source include/have_binlog_format_row.inc - +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); SET AUTOCOMMIT=0; # Create 1st table diff --git a/mysql-test/suite/binlog_encryption/binlog_write_error.result b/mysql-test/suite/binlog_encryption/binlog_write_error.result index 2606a9f40b3..46cc346bb70 100644 --- a/mysql-test/suite/binlog_encryption/binlog_write_error.result +++ b/mysql-test/suite/binlog_encryption/binlog_write_error.result @@ -9,6 +9,7 @@ DROP PROCEDURE IF EXISTS p2; DROP TRIGGER IF EXISTS tr1; DROP TRIGGER IF EXISTS tr2; DROP VIEW IF EXISTS v1, v2; +call mtr.add_suppression("Write to binary log failed: Error writing file"); # # Test injecting binlog write error when executing queries # diff --git a/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result b/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result index 388c8e67b68..944ad9331ad 100644 --- a/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result +++ b/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result @@ -1,6 +1,8 @@ include/master-slave.inc [connection master] call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +call mtr.add_suppression("Write to binary log failed: Multi-row statements required more than .max_binlog_stmt_cache_size"); +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); SET GLOBAL max_binlog_cache_size = 4096; SET GLOBAL binlog_cache_size = 4096; SET GLOBAL max_binlog_stmt_cache_size = 4096; diff --git a/mysql-test/suite/rpl/r/rpl_mdev-11092.result b/mysql-test/suite/rpl/r/rpl_mdev-11092.result index 90b809477b2..6a6e832b784 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev-11092.result +++ b/mysql-test/suite/rpl/r/rpl_mdev-11092.result @@ -1,7 +1,10 @@ include/master-slave.inc [connection master] call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); -call mtr.add_suppression("Slave SQL: The incident LOST_EVENTS occured on the master. .*"); +call mtr.add_suppression("Slave SQL: The incident LOST_EVENTS occured on the master"); +call mtr.add_suppression("Write to binary log failed: Multi-row statements required more than .max_binlog_stmt_cache_size"); +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); +"*********** Annotate Event write failure **************" SET GLOBAL max_binlog_cache_size = 4096; SET GLOBAL binlog_cache_size = 4096; SET GLOBAL max_binlog_stmt_cache_size = 4096; @@ -10,12 +13,76 @@ disconnect master; connect master,127.0.0.1,root,,test,$MASTER_MYPORT,; CREATE TABLE t1(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=MYISAM; connection master; -ERROR HY000: Writing one row to the row-based binary log failed +"#######################################################################" +"# Test Case1: Annotate event write failure for MyISAM #" +"#######################################################################" +ERROR HY000: Multi-row statements required more than 'max_binlog_stmt_cache_size' bytes of storage; increase this mysqld variable and try again include/wait_for_slave_sql_error_and_skip.inc [errno=1590] +"#######################################################################" +"# Test Case2: Annotate event write failure for INNODB #" +"#######################################################################" connection master; +CREATE TABLE t2(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=INNODB; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mysqld variable and try again +"*** One row will be present as second row will be rolledback ***" +SELECT COUNT(*) FROM t2; +COUNT(*) +1 +connection slave; +SELECT COUNT(*) FROM t2; +COUNT(*) +1 +"#######################################################################" +"# Test Case3: Annotate event write failure for mixed engine UPDATE #" +"#######################################################################" +connection master; +ERROR HY000: Multi-row statements required more than 'max_binlog_stmt_cache_size' bytes of storage; increase this mysqld variable and try again +include/wait_for_slave_sql_error_and_skip.inc [errno=1590] +connection master; +"****** Clean up *******" SET GLOBAL max_binlog_cache_size= ORIGINAL_VALUE; SET GLOBAL binlog_cache_size= ORIGINAL_VALUE; SET GLOBAL max_binlog_stmt_cache_size= ORIGINAL_VALUE; SET GLOBAL binlog_stmt_cache_size= ORIGINAL_VALUE; -DROP TABLE t1; +DROP TABLE t1,t2; +"*********** TABLE MAP Event write failure **************" +SET @save_debug= @@GLOBAL.debug_dbug; +CREATE TABLE tm (f INT) ENGINE=MYISAM; +CREATE TABLE ti (f INT) ENGINE=INNODB; +INSERT INTO tm VALUES (10); +INSERT INTO ti VALUES (20); +connection slave; +"#######################################################################" +"# Test Case4: Table_map event write failure for mixed engine UPDATE #" +"#######################################################################" +connection master; +SET debug_dbug="+d,table_map_write_error"; +"In case of mixed engines if non trans table is updated write INCIDENT event" +UPDATE ti,tm SET tm.f=88, ti.f=120; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mysqld variable and try again +include/wait_for_slave_sql_error_and_skip.inc [errno=1590] +"On Slave non trans table should be updated one row with f=88 should be found" +SELECT * FROM tm WHERE f=88; +f +88 +"On Innodb table 'ti' no update should be done no row exists with f=120" +SELECT COUNT(*) FROM ti WHERE f=120; +COUNT(*) +0 +"#######################################################################" +"# Test Case5: Table_map event write failure for trans engine UPDATE #" +"#######################################################################" +"Transaction will be rolled back. No incident event is written." +connection master; +UPDATE ti, tm set ti.f=30; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mysqld variable and try again +"Verify on master that no rows exists with f=30" +SELECT COUNT(*) FROM ti WHERE f=30; +COUNT(*) +0 +connection slave; +connection master; +"******** Clean Up **********" +SET GLOBAL debug_dbug= @save_debug; +DROP TABLE tm,ti; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_mixed_binlog_max_cache_size.result b/mysql-test/suite/rpl/r/rpl_mixed_binlog_max_cache_size.result index 388c8e67b68..944ad9331ad 100644 --- a/mysql-test/suite/rpl/r/rpl_mixed_binlog_max_cache_size.result +++ b/mysql-test/suite/rpl/r/rpl_mixed_binlog_max_cache_size.result @@ -1,6 +1,8 @@ include/master-slave.inc [connection master] call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +call mtr.add_suppression("Write to binary log failed: Multi-row statements required more than .max_binlog_stmt_cache_size"); +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); SET GLOBAL max_binlog_cache_size = 4096; SET GLOBAL binlog_cache_size = 4096; SET GLOBAL max_binlog_stmt_cache_size = 4096; diff --git a/mysql-test/suite/rpl/r/rpl_stm_binlog_max_cache_size.result b/mysql-test/suite/rpl/r/rpl_stm_binlog_max_cache_size.result index 388c8e67b68..944ad9331ad 100644 --- a/mysql-test/suite/rpl/r/rpl_stm_binlog_max_cache_size.result +++ b/mysql-test/suite/rpl/r/rpl_stm_binlog_max_cache_size.result @@ -1,6 +1,8 @@ include/master-slave.inc [connection master] call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +call mtr.add_suppression("Write to binary log failed: Multi-row statements required more than .max_binlog_stmt_cache_size"); +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); SET GLOBAL max_binlog_cache_size = 4096; SET GLOBAL binlog_cache_size = 4096; SET GLOBAL max_binlog_stmt_cache_size = 4096; diff --git a/mysql-test/suite/rpl/t/rpl_mdev-11092.test b/mysql-test/suite/rpl/t/rpl_mdev-11092.test index 31a385b40e6..7f6be80582d 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev-11092.test +++ b/mysql-test/suite/rpl/t/rpl_mdev-11092.test @@ -1,3 +1,4 @@ +--source include/have_debug.inc --source include/have_innodb.inc --source include/not_embedded.inc --source include/not_windows.inc @@ -6,13 +7,17 @@ ######################################################################################## call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); -call mtr.add_suppression("Slave SQL: The incident LOST_EVENTS occured on the master. .*"); +call mtr.add_suppression("Slave SQL: The incident LOST_EVENTS occured on the master"); +call mtr.add_suppression("Write to binary log failed: Multi-row statements required more than .max_binlog_stmt_cache_size"); +call mtr.add_suppression("Write to binary log failed: Multi-statement transaction required more than .max_binlog_cache_size"); let $old_max_binlog_cache_size= query_get_value(SHOW VARIABLES LIKE "max_binlog_cache_size", Value, 1); let $old_binlog_cache_size= query_get_value(SHOW VARIABLES LIKE "binlog_cache_size", Value, 1); let $old_max_binlog_stmt_cache_size= query_get_value(SHOW VARIABLES LIKE "max_binlog_stmt_cache_size", Value, 1); let $old_binlog_stmt_cache_size= query_get_value(SHOW VARIABLES LIKE "binlog_stmt_cache_size", Value, 1); +--echo "*********** Annotate Event write failure **************" + SET GLOBAL max_binlog_cache_size = 4096; SET GLOBAL binlog_cache_size = 4096; SET GLOBAL max_binlog_stmt_cache_size = 4096; @@ -26,8 +31,19 @@ let $data = `select concat('"', repeat('a',2000), '"')`; connection master; +# Insert a huge row into MyISAM table. The row will be inserted in engine and a +# request to write to binary log will be initiated. Since row annotations are +# enabled the size of the annotate event itself will exceed the +# "max_binlog_stmt_cache_size". This will result in ER_STMT_CACHE_FULL error +# and an incident event will be written to the binary log as row update in +# engine cannot be undone. + +--echo "#######################################################################" +--echo "# Test Case1: Annotate event write failure for MyISAM #" +--echo "#######################################################################" + --disable_query_log ---error ER_BINLOG_ROW_LOGGING_FAILED +--error ER_STMT_CACHE_FULL eval INSERT INTO t1 (a, data) VALUES (2, CONCAT($data, $data, $data, $data, $data, $data)); --enable_query_log @@ -37,8 +53,59 @@ eval INSERT INTO t1 (a, data) VALUES (2, --let $slave_sql_errno= 1590 --source include/wait_for_slave_sql_error_and_skip.inc -connection master; +# MDEV-21087 +# Insert two huge rows in to transaction cache. Have data such that first row +# fits inside the binary log cache. While writing the annotate event for the +# second row the binary log cache size will exceed "max_binlog_cache_size". +# Hence this statement cannot be written to binary log. As DMLs in Innodb can +# be safely rolled back only an error will be reported. Slave will continue to +# work. + +--echo "#######################################################################" +--echo "# Test Case2: Annotate event write failure for INNODB #" +--echo "#######################################################################" + +--connection master +CREATE TABLE t2(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=INNODB; +--disable_query_log +BEGIN; +eval INSERT INTO t2 (a, data) VALUES (1, CONCAT($data, $data)); +--error ER_TRANS_CACHE_FULL +eval INSERT INTO t2 (a, data) VALUES (2, CONCAT($data, $data)); +COMMIT; +--enable_query_log + +--echo "*** One row will be present as second row will be rolledback ***" +SELECT COUNT(*) FROM t2; +--sync_slave_with_master +SELECT COUNT(*) FROM t2; +# Testing mixed engine UPDATE statement scenario. In the following multi +# update query 'ha_update_row' will be invoked for t1 (myisam) table. This +# intern invokes binlog_write_table_map() function call. While writing a huge +# annotate event binary log cache size will exceed max_binlog_cache_size. +# Writing to binary log fails. Since non transactional changes cannot be +# rolled back incident event will be written to binary log. + +--echo "#######################################################################" +--echo "# Test Case3: Annotate event write failure for mixed engine UPDATE #" +--echo "#######################################################################" + +--connection master +let $new_data = `select concat('"', repeat('b',2000), '"')`; +--disable_query_log +--error ER_STMT_CACHE_FULL +eval UPDATE t1,t2 SET t1.data="Hello", t2.data=CONCAT($new_data,$new_data,$new_data,$new_data,$new_data); +--enable_query_log + +# Incident event +# 1590=ER_SLAVE_INCIDENT +--let $slave_sql_errno= 1590 +--source include/wait_for_slave_sql_error_and_skip.inc + +--connection master + +--echo "****** Clean up *******" --replace_result $old_max_binlog_cache_size ORIGINAL_VALUE --eval SET GLOBAL max_binlog_cache_size= $old_max_binlog_cache_size --replace_result $old_binlog_cache_size ORIGINAL_VALUE @@ -48,6 +115,52 @@ connection master; --replace_result $old_binlog_stmt_cache_size ORIGINAL_VALUE --eval SET GLOBAL binlog_stmt_cache_size= $old_binlog_stmt_cache_size -DROP TABLE t1; +DROP TABLE t1,t2; + +--echo "*********** TABLE MAP Event write failure **************" + +SET @save_debug= @@GLOBAL.debug_dbug; +CREATE TABLE tm (f INT) ENGINE=MYISAM; +CREATE TABLE ti (f INT) ENGINE=INNODB; +INSERT INTO tm VALUES (10); +INSERT INTO ti VALUES (20); +--sync_slave_with_master + +--echo "#######################################################################" +--echo "# Test Case4: Table_map event write failure for mixed engine UPDATE #" +--echo "#######################################################################" +--connection master +SET debug_dbug="+d,table_map_write_error"; +--echo "In case of mixed engines if non trans table is updated write INCIDENT event" +--error ER_TRANS_CACHE_FULL +UPDATE ti,tm SET tm.f=88, ti.f=120; + +# Incident event +# 1590=ER_SLAVE_INCIDENT +--let $slave_sql_errno= 1590 +--source include/wait_for_slave_sql_error_and_skip.inc + +--echo "On Slave non trans table should be updated one row with f=88 should be found" +SELECT * FROM tm WHERE f=88; + +--echo "On Innodb table 'ti' no update should be done no row exists with f=120" +SELECT COUNT(*) FROM ti WHERE f=120; + +--echo "#######################################################################" +--echo "# Test Case5: Table_map event write failure for trans engine UPDATE #" +--echo "#######################################################################" +--echo "Transaction will be rolled back. No incident event is written." +--connection master +--error ER_TRANS_CACHE_FULL +UPDATE ti, tm set ti.f=30; +--echo "Verify on master that no rows exists with f=30" +SELECT COUNT(*) FROM ti WHERE f=30; + +--sync_slave_with_master + +--connection master +--echo "******** Clean Up **********" +SET GLOBAL debug_dbug= @save_debug; +DROP TABLE tm,ti; --source include/rpl_end.inc diff --git a/sql/handler.cc b/sql/handler.cc index 914a4dc07b1..6cd5ab15b36 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -5863,7 +5863,27 @@ static int write_locked_table_maps(THD *thd) continue; TABLE **const end_ptr= lock->table + lock->table_count; - for (TABLE **table_ptr= lock->table ; + bool stmt_modified_non_trans_table= false; + /* + Iterate through list of tables and identify if multi statement + transaction modifies any non transactional table. Set the + 'stmt_modified_non_trans_table' flag to 'true'. In case if there a + failure at the time of writing annotate/table_map event into the binary + log this flag will enable server to write an incident event into the + binary log. This additional flag is required as + 'thd->transaction.stmt.modified_non_trans_table' will not available at + the time of writing table_map/annotate event. + */ + for (TABLE **table_ptr= lock->table; table_ptr != end_ptr ; ++table_ptr) + { + TABLE *const table= *table_ptr; + if (!table->file->has_transactions() && table->current_lock == F_WRLCK) + { + stmt_modified_non_trans_table=true; + break; + } + } + for (TABLE **table_ptr= lock->table ; table_ptr != end_ptr ; ++table_ptr) { @@ -5887,8 +5907,10 @@ static int write_locked_table_maps(THD *thd) */ bool const has_trans= thd->lex->sql_command == SQLCOM_CREATE_TABLE || table->file->has_transactions(); + int const error= thd->binlog_write_table_map(table, has_trans, - &with_annotate); + &with_annotate, + stmt_modified_non_trans_table); /* If an error occurs, it is the responsibility of the caller to roll back the transaction. diff --git a/sql/log.cc b/sql/log.cc index 5fd384d55a0..a82808bc7b7 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -5673,14 +5673,14 @@ binlog_start_consistent_snapshot(handlerton *hton, THD *thd) } /** - This function writes a table map to the binary log. + This function writes a table map to the binary log. Note that in order to keep the signature uniform with related methods, we use a redundant parameter to indicate whether a transactional table was changed or not. If with_annotate != NULL and *with_annotate = TRUE write also Annotate_rows before the table map. - + @param table a pointer to the table. @param is_transactional @c true indicates a transactional table, otherwise @c false a non-transactional. @@ -5688,7 +5688,8 @@ binlog_start_consistent_snapshot(handlerton *hton, THD *thd) nonzero if an error pops up when writing the table map event. */ int THD::binlog_write_table_map(TABLE *table, bool is_transactional, - my_bool *with_annotate) + my_bool *with_annotate, + bool stmt_modified_non_trans_table) { int error; DBUG_ENTER("THD::binlog_write_table_map"); @@ -5699,7 +5700,7 @@ int THD::binlog_write_table_map(TABLE *table, bool is_transactional, /* Ensure that all events in a GTID group are in the same cache */ if (variables.option_bits & OPTION_GTID_BEGIN) is_transactional= 1; - + /* Pre-conditions */ DBUG_ASSERT(is_current_stmt_binlog_format_row()); DBUG_ASSERT(WSREP_EMULATE_BINLOG(this) || mysql_bin_log.is_open()); @@ -5726,17 +5727,34 @@ int THD::binlog_write_table_map(TABLE *table, bool is_transactional, /* Annotate event should be written not more than once */ *with_annotate= 0; if ((error= writer.write(&anno))) - { - if (my_errno == EFBIG) - cache_data->set_incident(); - DBUG_RETURN(error); - } + goto write_err; } + DBUG_EXECUTE_IF("table_map_write_error", + { + if (is_transactional) + { + my_errno= EFBIG; + error= 1; + goto write_err; + } + }); if ((error= writer.write(&the_event))) - DBUG_RETURN(error); + goto write_err; - binlog_table_maps++; - DBUG_RETURN(0); + binlog_table_maps++; + DBUG_RETURN(0); + +write_err: + mysql_bin_log.set_write_error(this, is_transactional); + /* + For non-transactional engine or multi statement transaction with mixed + engines, data is written to table but writing to binary log failed. In + these scenarios rollback is not possible. Hence report an incident. + */ + if (mysql_bin_log.check_write_error(this) && cache_data && + stmt_modified_non_trans_table) + cache_data->set_incident(); + DBUG_RETURN(error); } /** @@ -7198,6 +7216,18 @@ bool MYSQL_BIN_LOG::write_incident(THD *thd) mysql_mutex_unlock(&LOCK_log); } + /* + Upon writing incident event, check for thd->error() and print the + relevant error message in the error log. + */ + if (!error && thd->is_error()) + { + sql_print_error("Write to binary log failed: " + "%s. An incident event is written to binary log " + "and slave will be stopped.\n", + thd->get_stmt_da()->message()); + } + DBUG_RETURN(error); } diff --git a/sql/sql_class.h b/sql/sql_class.h index b35f9a93238..133bc9de607 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -2423,7 +2423,8 @@ class THD :public Statement, void binlog_start_trans_and_stmt(); void binlog_set_stmt_begin(); int binlog_write_table_map(TABLE *table, bool is_transactional, - my_bool *with_annotate= 0); + my_bool *with_annotate= 0, + bool stmt_modified_non_trans_table= false); int binlog_write_row(TABLE* table, bool is_transactional, const uchar *buf); int binlog_delete_row(TABLE* table, bool is_transactional,

1 0

[Commits] 0ff7c7974dc: MDEV-18514: Assertion `!writer.checksum_len || writer.remains == 0' failed
by sujatha 08 Jan '20

08 Jan '20

revision-id: 0ff7c7974dc496caaab38eebd9896fd14f6318e2 (mariadb-10.2.30-39-g0ff7c7974dc) parent(s): 16bce0f6fe6bcad0091dc45a97a8ac7b33fe9d44 author: Sujatha committer: Sujatha timestamp: 2020-01-08 13:01:38 +0530 message: MDEV-18514: Assertion `!writer.checksum_len || writer.remains == 0' failed Analysis: ======== 'max_binlog_cache_size' is configured and a huge transaction is executed. When the transaction specific events size exceeds 'max_binlog_cache_size' the event cannot be written to the binary log cache and cache write error is raised. Upon cache write error the statement is rolled back and the transaction cache should be truncated to a previous statement specific position. The truncate operation should reset the cache to earlier valid positions and flush the new changes. Even though the flush is successful the cache write error is still in marked state. The truncate code interprets the cache write error as cache flush failure and returns abruptly without modifying the write cache parameters. Hence cache is in a invalid state. When a COMMIT statement is executed in this session it tries to flush the contents of transaction cache to binary log. Since cache has partial events the cache write operation will report 'writer.remains' assert. Fix: === Binlog truncate function resets the cache to a specified size. As a first step of truncation, clear the cache write error flag that was raised during earlier execution. With this new errors that surface during cache truncation can be clearly identified. --- .../suite/rpl/r/rpl_binlog_rollback_cleanup.result | 9 +++++ .../suite/rpl/t/rpl_binlog_rollback_cleanup.test | 46 ++++++++++++++++++++++ sql/log.cc | 3 +- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/rpl/r/rpl_binlog_rollback_cleanup.result b/mysql-test/suite/rpl/r/rpl_binlog_rollback_cleanup.result new file mode 100644 index 00000000000..a677cbfecf6 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_binlog_rollback_cleanup.result @@ -0,0 +1,9 @@ +include/master-slave.inc +[connection master] +connection master; +SET GLOBAL max_binlog_cache_size = 65536; +CREATE TABLE t1(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=INNODB; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mysqld variable and try again +SET GLOBAL max_binlog_cache_size= ORIGINAL_VALUE; +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_binlog_rollback_cleanup.test b/mysql-test/suite/rpl/t/rpl_binlog_rollback_cleanup.test new file mode 100644 index 00000000000..ed4d713f626 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_binlog_rollback_cleanup.test @@ -0,0 +1,46 @@ +# ==== Purpose ==== +# +# Test verifies that when flushing an event to binary log fails the transaction +# is successfully rolled back and following COMMIT command doesn't report any +# assert. +# +# ==== Implementation ==== +# +# Steps: +# 0 - SET max_binlog_cache_size=64K +# 1 - Create an Innodb table and insert required amount of data. Execute an +# UPDATE operation which generates a big update event whose size exceeds +# max_binlog_cache_size. +# 2 - Wait for error 1197. Execute COMMIT command. +# 3 - COMMIT should be successful. +# +# ==== References ==== +# +# MDEV-18514: Assertion `!writer.checksum_len || writer.remains == 0' failed +# +--source include/have_innodb.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc +--connection master +let $old_max_binlog_cache_size= query_get_value(SHOW VARIABLES LIKE "max_binlog_cache_size", Value, 1); +SET GLOBAL max_binlog_cache_size = 65536; +CREATE TABLE t1(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=INNODB; +let $data = `select concat('"', repeat('a',6000), '"')`; +let $data1 = `select concat('"', repeat('b',6000), '"')`; +--disable_query_log +eval INSERT INTO t1 (a, data) VALUES (1, CONCAT($data, $data)); +eval INSERT INTO t1 (a, data) VALUES (2, CONCAT($data, $data)); +eval INSERT INTO t1 (a, data) VALUES (3, CONCAT($data, $data)); +eval INSERT INTO t1 (a, data) VALUES (4, CONCAT($data, $data)); +eval INSERT INTO t1 (a, data) VALUES (5, CONCAT($data, $data)); +START TRANSACTION; +--error ER_TRANS_CACHE_FULL +eval UPDATE t1 SET data=$data1; +COMMIT; +--enable_query_log + +--replace_result $old_max_binlog_cache_size ORIGINAL_VALUE +--eval SET GLOBAL max_binlog_cache_size= $old_max_binlog_cache_size +DROP TABLE t1; + +--source include/rpl_end.inc diff --git a/sql/log.cc b/sql/log.cc index 5fd384d55a0..91dfac07993 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -448,6 +448,7 @@ class binlog_cache_data void truncate(my_off_t pos) { DBUG_PRINT("info", ("truncating to position %lu", (ulong) pos)); + cache_log.error=0; if (pending()) { delete pending(); @@ -456,7 +457,7 @@ class binlog_cache_data reinit_io_cache(&cache_log, WRITE_CACHE, pos, 0, 0); cache_log.end_of_file= saved_max_binlog_cache_size; } - + binlog_cache_data& operator=(const binlog_cache_data& info); binlog_cache_data(const binlog_cache_data& info); };

1 0

[Commits] 6a43a1f: More updates
by psergey 07 Jan '20

07 Jan '20

revision-id: 6a43a1f9975ba6851223db33b64959ad21f6cd6d () parent(s): 7f8b19cc3f261edb5beb446cef994cd65403a9bf author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-07 23:31:18 +0300 message: More updates --- run-mysql-8.0.sh | 1 + test-harness/schema-winkyao.sql | 42 ++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/run-mysql-8.0.sh b/run-mysql-8.0.sh index b6a00a9..8f69cac 100644 --- a/run-mysql-8.0.sh +++ b/run-mysql-8.0.sh @@ -1,5 +1,6 @@ bash setup-server/setup-mysql-8.0.sh bash test-harness/01-setup.sh bash test-harness/10-load-data.sh +bash test-harness/mysql-50-eits-analyze.sh bash test-harness/20-generate-workload.sh bash test-harness/30-run-workload.sh diff --git a/test-harness/schema-winkyao.sql b/test-harness/schema-winkyao.sql index ad997ae..c47c543 100644 --- a/test-harness/schema-winkyao.sql +++ b/test-harness/schema-winkyao.sql @@ -14,7 +14,7 @@ CREATE TABLE aka_name ( name_pcode_nf varchar(11), surname_pcode varchar(11), md5sum varchar(65) -); +) character set utf8 ; CREATE TABLE aka_title ( id integer NOT NULL PRIMARY KEY, @@ -29,7 +29,7 @@ CREATE TABLE aka_title ( episode_nr integer, note varchar(72), md5sum varchar(32) -); +) character set utf8 ; -- psergey: changed "note text" to "note varchar(1000)" below: CREATE TABLE cast_info ( @@ -40,7 +40,7 @@ CREATE TABLE cast_info ( note varchar(1000), nr_order integer, role_id integer NOT NULL -); +) character set utf8 ; CREATE TABLE char_name ( id integer NOT NULL PRIMARY KEY, @@ -50,12 +50,12 @@ CREATE TABLE char_name ( name_pcode_nf varchar(5), surname_pcode varchar(5), md5sum varchar(32) -); +) character set utf8 ; CREATE TABLE comp_cast_type ( id integer NOT NULL PRIMARY KEY, kind varchar(32) NOT NULL -); +) character set utf8 ; CREATE TABLE company_name ( id integer NOT NULL PRIMARY KEY, @@ -65,40 +65,40 @@ CREATE TABLE company_name ( name_pcode_nf varchar(5), name_pcode_sf varchar(5), md5sum varchar(32) -); +) character set utf8 ; CREATE TABLE company_type ( id integer NOT NULL PRIMARY KEY, kind varchar(32) -); +) character set utf8 ; CREATE TABLE complete_cast ( id integer NOT NULL PRIMARY KEY, movie_id integer, subject_id integer NOT NULL, status_id integer NOT NULL -); +) character set utf8 ; CREATE TABLE info_type ( id integer NOT NULL PRIMARY KEY, info varchar(32) NOT NULL -); +) character set utf8 ; CREATE TABLE keyword ( id integer NOT NULL PRIMARY KEY, keyword varchar(512) NOT NULL, phonetic_code varchar(5) -); +) character set utf8 ; CREATE TABLE kind_type ( id integer NOT NULL PRIMARY KEY, kind varchar(15) -); +) character set utf8 ; CREATE TABLE link_type ( id integer NOT NULL PRIMARY KEY, link varchar(32) NOT NULL -); +) character set utf8 ; CREATE TABLE movie_companies ( id integer NOT NULL PRIMARY KEY, @@ -106,7 +106,7 @@ CREATE TABLE movie_companies ( company_id integer NOT NULL, company_type_id integer NOT NULL, note text -); +) character set utf8 ; CREATE TABLE movie_info_idx ( id integer NOT NULL PRIMARY KEY, @@ -114,20 +114,20 @@ CREATE TABLE movie_info_idx ( info_type_id integer NOT NULL, info text NOT NULL, note text -); +) character set utf8 ; CREATE TABLE movie_keyword ( id integer NOT NULL PRIMARY KEY, movie_id integer NOT NULL, keyword_id integer NOT NULL -); +) character set utf8 ; CREATE TABLE movie_link ( id integer NOT NULL PRIMARY KEY, movie_id integer NOT NULL, linked_movie_id integer NOT NULL, link_type_id integer NOT NULL -); +) character set utf8 ; CREATE TABLE name ( id integer NOT NULL PRIMARY KEY, @@ -139,12 +139,12 @@ CREATE TABLE name ( name_pcode_nf varchar(5), surname_pcode varchar(5), md5sum varchar(32) -); +) character set utf8 ; CREATE TABLE role_type ( id integer NOT NULL PRIMARY KEY, role varchar(32) NOT NULL -); +) character set utf8; CREATE TABLE title ( id integer NOT NULL PRIMARY KEY, @@ -159,7 +159,7 @@ CREATE TABLE title ( episode_nr integer, series_years varchar(49), md5sum varchar(32) -); +) character set utf8; CREATE TABLE movie_info ( id integer NOT NULL PRIMARY KEY, @@ -167,7 +167,7 @@ CREATE TABLE movie_info ( info_type_id integer NOT NULL, info text NOT NULL, note text -); +) character set utf8; CREATE TABLE person_info ( id integer NOT NULL PRIMARY KEY, @@ -175,4 +175,4 @@ CREATE TABLE person_info ( info_type_id integer NOT NULL, info text NOT NULL, note text -); +) character set utf8;

1 0

[Commits] 7f8b19c: More updates
by psergey 07 Jan '20

07 Jan '20

revision-id: 7f8b19cc3f261edb5beb446cef994cd65403a9bf () parent(s): 68d519ee26c75a39009046d8033199d5636e6cfb author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-07 22:13:10 +0300 message: More updates --- .gitignore | 7 +++++++ test-harness/000-setup-dataset.sh | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index 0dd33f0..7f6774e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,10 @@ mysql-boost mysql-8.0 mysql8-data/ tmp-mysql +pgdata +postgresql-12.1-inst +postgresql-12.1 +postgresql-12.1.tar.bz2 +psql-vars.sh +tmp-postgresql +imdb-2014-csv-postgresql diff --git a/test-harness/000-setup-dataset.sh b/test-harness/000-setup-dataset.sh index bd2cfa3..1cc5a91 100644 --- a/test-harness/000-setup-dataset.sh +++ b/test-harness/000-setup-dataset.sh @@ -5,12 +5,16 @@ if [ ! -d /optane/data/imdb-2014-mysql ] ; then (cd imdb-2014-csv-postgresql; wget http://homepages.cwi.nl/~boncz/job/imdb.tgz; tar zxvf imdb.tgz) +else + cp -r /optane/data/imdb-2014-postgresql imdb-2014-csv-postgresql fi if [ ! -d postgresql-12.1-inst ] ; then bash setup-server/setup-postgresql-12.sh fi +mkdir -p tmp-postgresql + bash test-harness/pg-10-load-data.sh mkdir -p ./imdb-2014-csv-mysql

1 0

[Commits] 078ed1c: Updates to run with 2 tables each using different CFs.
by psergey 07 Jan '20

07 Jan '20

revision-id: 078ed1c21b65fc4a8fab19f72c7d8cc3a68c27aa () parent(s): 4fbad6cc8f98d1fb841c9cea6c930c4c70cb03e6 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-07 17:11:44 +0300 message: Updates to run with 2 tables each using different CFs. --- make-2-cfs.sql | 26 ++++++++++++++++++++++++++ run-all.sh | 4 ++-- run-test.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 75 insertions(+), 14 deletions(-) diff --git a/make-2-cfs.sql b/make-2-cfs.sql new file mode 100644 index 0000000..67f0d0d --- /dev/null +++ b/make-2-cfs.sql @@ -0,0 +1,26 @@ +use sbtest; +alter table sbtest1 rename sbtest1_orig; +alter table sbtest2 rename sbtest2_orig; + +CREATE TABLE sbtest1 ( + id int(11) NOT NULL AUTO_INCREMENT, + k int(11) NOT NULL DEFAULT '0', + c char(120) NOT NULL DEFAULT '', + pad char(60) NOT NULL DEFAULT '', + PRIMARY KEY (id) COMMENT 'cf1_pk', + KEY k_1 (k) COMMENT 'cf1_sk' +) ENGINE=ROCKSDB CHARSET=latin1; + +CREATE TABLE sbtest2 ( + id int(11) NOT NULL AUTO_INCREMENT, + k int(11) NOT NULL DEFAULT '0', + c char(120) NOT NULL DEFAULT '', + pad char(60) NOT NULL DEFAULT '', + PRIMARY KEY (id) COMMENT 'cf2_pk', + KEY k_1 (k) COMMENT 'cf2_sk' +) ENGINE=ROCKSDB CHARSET=latin1; + +set rocksdb_commit_in_the_middle=1; +insert into sbtest1 select * from sbtest1_orig; +insert into sbtest2 select * from sbtest2_orig; +set rocksdb_commit_in_the_middle=0; diff --git a/run-all.sh b/run-all.sh index eb897b5..55bb595 100755 --- a/run-all.sh +++ b/run-all.sh @@ -4,8 +4,8 @@ echo "rocksdb_use_range_locking=1" >> my-fbmysql-range-locking.cnf -./run-test.sh orig 01-orig -./run-test.sh range-locking 02-range-locking +./run-test.sh -m orig 01-orig +./run-test.sh -m range-locking 02-range-locking ./summarize-result.sh 01-orig ./summarize-result.sh 02-range-locking diff --git a/run-test.sh b/run-test.sh index 21411a0..5871a3a 100755 --- a/run-test.sh +++ b/run-test.sh @@ -1,15 +1,18 @@ #!/bin/bash +usage () { +echo "Usage: $0 [-p] [-m] [-c] [-d] server_name test_run_name" +echo " -p - Use perf for profiling" +echo " -m - Put datadir on /dev/shm" +echo " -c - Assume sysbench uses 4 tables and move them to different CFs." +echo " -d - Same but use 2 CFs" +} + ### ### Parse options ### -USAGE_STR="Usage: $0 [-p] [-m] [-c] server_name test_run_name \ - -p - Use perf for profiling - -m - Put datadir on /dev/shm - -c - Assume sysbench uses 4 tables and move them to different CFs. -" -while getopts ":pmc" opt; do +while getopts ":pmcd" opt; do case ${opt} in p ) USE_PERF=1 ;; @@ -17,8 +20,11 @@ while getopts ":pmc" opt; do ;; c ) USE_4_CFS=1 ;; - \? ) echo $USAGE_STR - exit 1 + d ) USE_2_CFS=1 + ;; + \? ) + usage; + exit 1 ;; esac done @@ -27,13 +33,18 @@ shift $((OPTIND -1)) SERVERNAME=$1 RUN_NAME=$2 +if [[ $USE_4_CFS && $USE_2_CFS ]] ; then + echo "Use either -c (USE_4_CFS) or -d (USE_2_CFS)" + exit 1 +fi + if [ "x${SERVERNAME}y" = "xy" ] ; then - echo $USAGE_STR + usage; exit 1 fi if [ "x${RUN_NAME}y" = "xy" ] ; then - echo $USAGE_STR + usage; exit 1 fi @@ -58,7 +69,14 @@ fi if [[ $USE_RAMDISK ]] ; then echo " Using /dev/shm for data dir" fi -#exit 0 + +if [[ $USE_4_CFS ]] ; then + echo " Data is in 4 tables" +fi + +if [[ $USE_2_CFS ]] ; then + echo " Data is in 2 tables" +fi ############################################################################# ### Start the server @@ -120,7 +138,17 @@ mkdir -p $RESULT_DIR SYSBENCH_ARGS=" --db-driver=mysql --mysql-host=127.0.0.1 --mysql-user=root \ --mysql-storage-engine=rocksdb \ --time=60 \ - /usr/share/sysbench/oltp_write_only.lua --table-size=250000 --tables=4" + /usr/share/sysbench/oltp_write_only.lua" + +if [[ $USE_4_CFS ]] ; then + SYSBENCH_ARGS="$SYSBENCH_ARGS --tables=4 --table-size=250000" +elif [[ $USE_2_CFS ]] ; then + SYSBENCH_ARGS="$SYSBENCH_ARGS --tables=2 --table-size=500000" +else + SYSBENCH_ARGS="$SYSBENCH_ARGS --table-size=1000000" +fi + +## /usr/share/sysbench/oltp_write_only.lua --table-size=250000 --tables=4" SYSBENCH_TEST="oltp_write_only.lua" cat > $RESULT_DIR/info.txt <<END @@ -139,6 +167,13 @@ if [[ $USE_4_CFS ]] ; then ./$SERVER_DIR/client/mysql --defaults-file=./my-fbmysql-${SERVERNAME}.cnf \ -uroot < make-4-cfs.sql fi + +if [[ $USE_2_CFS ]] ; then + echo "Splitting 2 tables into different CFs" + ./$SERVER_DIR/client/mysql --defaults-file=./my-fbmysql-${SERVERNAME}.cnf \ + -uroot < make-2-cfs.sql +fi + sleep 3 ./$SERVER_DIR/client/mysql --defaults-file=./my-fbmysql-${SERVERNAME}.cnf \ -uroot -e "show variables like 'rocksdb%'" > $RESULT_DIR/variables-rocksdb.txt

1 0

[Commits] 3b3b49b0049: Count the obtained range locks in Rdb_transaction::m_lock_count
by psergey 06 Jan '20

06 Jan '20

revision-id: 3b3b49b00496de0807646583d47425b7718fcaa6 (fb-prod201903-275-g3b3b49b0049) parent(s): 6c2479afa9280fc5ac6d7f904a9a8dca284c6f5b author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-01-06 22:37:43 +0300 message: Count the obtained range locks in Rdb_transaction::m_lock_count This fixes rocksdb.trx_info'range_locking' test. --- storage/rocksdb/ha_rocksdb.cc | 4 +++- storage/rocksdb/rdb_locking_iter.cc | 6 ++++-- storage/rocksdb/rdb_locking_iter.h | 11 ++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 44ab3ef36df..c2fbaa25293 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3378,6 +3378,7 @@ class Rdb_transaction_impl : public Rdb_transaction { rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Endpoint &start_endp, const rocksdb::Endpoint &end_endp) override { + ++m_lock_count; return m_rocksdb_tx->GetRangeLock(cf, start_endp, end_endp); } private: @@ -3622,7 +3623,8 @@ class Rdb_transaction_impl : public Rdb_transaction { global_stats.queries[QUERIES_RANGE].inc(); if (use_locking_iterator) { locking_iter_created(); - return GetLockingIterator(m_rocksdb_tx, options, column_family, is_rev_cf); + return GetLockingIterator(m_rocksdb_tx, options, column_family, + is_rev_cf, &m_lock_count); } else return m_rocksdb_tx->GetIterator(options, column_family); diff --git a/storage/rocksdb/rdb_locking_iter.cc b/storage/rocksdb/rdb_locking_iter.cc index 7f5e241d906..983d98639aa 100644 --- a/storage/rocksdb/rdb_locking_iter.cc +++ b/storage/rocksdb/rdb_locking_iter.cc @@ -12,8 +12,10 @@ rocksdb::Iterator* GetLockingIterator( rocksdb::Transaction *trx, const rocksdb::ReadOptions& read_options, rocksdb::ColumnFamilyHandle* column_family, - bool is_rev_cf) { - return new LockingIterator(trx, column_family, is_rev_cf, read_options); + bool is_rev_cf, + ulonglong *counter) { + return new LockingIterator(trx, column_family, is_rev_cf, read_options, + counter); } /* diff --git a/storage/rocksdb/rdb_locking_iter.h b/storage/rocksdb/rdb_locking_iter.h index 855f3f2cc0a..e77e5b77913 100644 --- a/storage/rocksdb/rdb_locking_iter.h +++ b/storage/rocksdb/rdb_locking_iter.h @@ -42,14 +42,17 @@ class LockingIterator : public rocksdb::Iterator { // note: an iterator that has reached EOF has status()==OK && valid_==false bool valid_; + ulonglong *lock_count_; public: LockingIterator(rocksdb::Transaction *txn, rocksdb::ColumnFamilyHandle *cfh, bool is_rev_cf, - const rocksdb::ReadOptions& opts + const rocksdb::ReadOptions& opts, + ulonglong *lock_count=nullptr ) : txn_(txn), cfh_(cfh), m_is_rev_cf(is_rev_cf), read_opts_(opts), iter_(nullptr), - status_(rocksdb::Status::InvalidArgument()), valid_(false) {} + status_(rocksdb::Status::InvalidArgument()), valid_(false), + lock_count_(lock_count) {} ~LockingIterator() { delete iter_; @@ -117,6 +120,7 @@ class LockingIterator : public rocksdb::Iterator { valid_ = false; return; } + if (lock_count_) (*lock_count_)++; std::string end_key_copy= end_key.ToString(); //Ok, now we have a lock which is inhibiting modifications in the range @@ -179,6 +183,7 @@ rocksdb::Iterator* GetLockingIterator(rocksdb::Transaction *trx, const rocksdb::ReadOptions& read_options, rocksdb::ColumnFamilyHandle* column_family, - bool is_rev_cf); + bool is_rev_cf, + ulonglong *counter); } // namespace myrocks

1 0