- commits - lists.mariadb.org

[Commits] 41893792e3d: Get first variant of fix for Issue #904
by Sergei Petrunia 05 Jan '19

05 Jan '19

revision-id: 41893792e3decc68eb68f952e2bc7239bb6e4d36 (fb-prod201801-181-g41893792e3d) parent(s): 9c0e1c912bf37cec8867806e0a7e46033668a99a author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-01-05 19:06:41 +0300 message: Get first variant of fix for Issue #904 Issue #904: Crash in rocksdb::IOStatsContext::Reset, this=NULL Fix both code paths: - Change the test source code so it doesn't cause the "Unused variable" warning (which -Werror converted into error and caused CMake not to set HAVE_THREAD_LOCAL) - If the system doesn't seem to support HAVE_THREAD_LOCAL, refuse to compile (rather than producing a binary that crashes for some tests) --- storage/rocksdb/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index 1a5357a2dea..e1d13efd81f 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -47,10 +47,15 @@ CHECK_CXX_SOURCE_COMPILES(" #endif int main() { static __thread int tls; + tls=0; + return tls; } " HAVE_THREAD_LOCAL) if(HAVE_THREAD_LOCAL) ADD_DEFINITIONS(-DROCKSDB_SUPPORT_THREAD_LOCAL) +else() + MESSAGE(SEND_ERROR "The compiler failed the check for ROCKSDB_SUPPORT_THREAD_LOCAL. " + "MyRocks requires that feature.") endif()

1 0

[Commits] 0c20b247de0: Disable galera.query_cache test.
by jan 04 Jan '19

04 Jan '19

revision-id: 0c20b247de0c1cc7f993e1eba6c4e3367c20e2c4 (mariadb-10.2.21-10-g0c20b247de0) parent(s): b6b15de85ddfa29762c5aa65128f36af0ad3b99e author: Jan Lindström committer: Jan Lindström timestamp: 2019-01-04 21:22:41 +0200 message: Disable galera.query_cache test. --- mysql-test/suite/galera/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 2e9d2a216ab..b7997627fff 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -47,4 +47,4 @@ galera.MW-328B : MDEV-17847 Galera test failure on MW-328[A|B|C] galera.MW-328C : MDEV-17847 Galera test failure on MW-328[A|B|C] galera.galera_sst_xtrabackup-v2 : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir] galera.galera_sst_xtrabackup-v2_data_dir : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir] -galera_query_cache : +query_cache : MDEV-18137: Galera test failure on query_cache

1 0

[Commits] b6b15de85dd: Disable failing Galera test galera_query_cache.
by jan 04 Jan '19

04 Jan '19

revision-id: b6b15de85ddfa29762c5aa65128f36af0ad3b99e (mariadb-10.2.21-9-gb6b15de85dd) parent(s): 3d8011b375a54552a540caecfc82d2d7a4059633 author: Jan Lindström committer: Jan Lindström timestamp: 2019-01-04 20:23:13 +0200 message: Disable failing Galera test galera_query_cache. --- mysql-test/suite/galera/disabled.def | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index c5fa1491d34..2e9d2a216ab 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -47,3 +47,4 @@ galera.MW-328B : MDEV-17847 Galera test failure on MW-328[A|B|C] galera.MW-328C : MDEV-17847 Galera test failure on MW-328[A|B|C] galera.galera_sst_xtrabackup-v2 : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir] galera.galera_sst_xtrabackup-v2_data_dir : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir] +galera_query_cache :

1 0

[Commits] 9793ff5671f: Disable failing galera test.
by jan 04 Jan '19

04 Jan '19

revision-id: 9793ff5671f74065329f5a82b16f9f70998f95f9 (mariadb-10.2.21-2-g9793ff5671f) parent(s): b78dc58dee9a231fd45e2b5835008091f16e20cc author: Jan Lindström committer: Jan Lindström timestamp: 2019-01-04 20:13:38 +0200 message: Disable failing galera test. --- mysql-test/suite/galera/disabled.def | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index c5fa1491d34..a454365c12d 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -47,3 +47,4 @@ galera.MW-328B : MDEV-17847 Galera test failure on MW-328[A|B|C] galera.MW-328C : MDEV-17847 Galera test failure on MW-328[A|B|C] galera.galera_sst_xtrabackup-v2 : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir] galera.galera_sst_xtrabackup-v2_data_dir : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir] +galera.query_cache :

1 0

[Commits] 9c0e1c912bf: MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys
by Sergei Petrunia 04 Jan '19

04 Jan '19

revision-id: 9c0e1c912bf37cec8867806e0a7e46033668a99a (fb-prod201801-180-g9c0e1c912bf) parent(s): bb04a774561548d159576b3bcaa2f638ffd76829 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-01-04 18:39:03 +0300 message: MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys In order to work correctly with multi-component indexes, Range Locking should use "Range endpoints" not lookup keys of range bounds. --- mysql-test/suite/rocksdb/r/range_locking.result | 50 ++++++- mysql-test/suite/rocksdb/r/varbinary_format.result | 17 ++- mysql-test/suite/rocksdb/t/delete_before_lock.test | 5 + mysql-test/suite/rocksdb/t/range_locking.test | 36 +++++ mysql-test/suite/rocksdb/t/varbinary_format.test | 14 +- rocksdb | 2 +- storage/rocksdb/ha_rocksdb.cc | 166 ++++++++++++++++++--- storage/rocksdb/range_locking/ft/comparator.h | 16 +- 8 files changed, 262 insertions(+), 44 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index 964458b783f..c3271db2730 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -90,18 +90,60 @@ pk a 10 10 select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 000001078000000a X +0 15 00000001078000000a X delete from t1 where pk between 25 and 40; select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 0000010780000019 - 0000010780000028 X -0 15 000001078000000a X +0 15 000000010780000019 - 010000010780000028 X +0 15 00000001078000000a X rollback; begin; +# The following will show a range lock on 2-9 and also a point lock on 10. +# This is how things currently work. select * from t1 where pk between 2 and 9 for update; pk a select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 16 0000010780000002 - 0000010780000009 X +0 16 00000001078000000a X +0 16 000000010780000002 - 010000010780000009 X rollback; drop table t1; +# +# MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( +kp1 int not null, +kp2 int not null, +a int, +primary key(kp1, kp2) +) engine=rocksdb; +insert into t1 select 1, a, 1234 from t0; +insert into t1 select 2, a, 1234 from t0; +insert into t1 select 3, a, 1234 from t0; +connect con1,localhost,root,,; +connection con1; +begin; +select * from t1 where kp1=2 for update; +kp1 kp2 a +2 0 1234 +2 1 1234 +2 2 1234 +2 3 1234 +2 4 1234 +2 5 1234 +2 6 1234 +2 7 1234 +2 8 1234 +2 9 1234 +connection default; +# The lock on kp1=2 should inhibit the following INSERT: +insert into t1 values ( 2,5,9999); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY +rollback; +connection con1; +rollback; +connection default; +disconnect con1; +drop table t0,t1; diff --git a/mysql-test/suite/rocksdb/r/varbinary_format.result b/mysql-test/suite/rocksdb/r/varbinary_format.result index 9362d42515c..e3dde737f9f 100644 --- a/mysql-test/suite/rocksdb/r/varbinary_format.result +++ b/mysql-test/suite/rocksdb/r/varbinary_format.result @@ -1,3 +1,4 @@ +set @skip_head=9 + if(@@rocksdb_use_range_locking, 2, 0); CREATE TABLE t1( vb VARBINARY(64) primary key ) ENGINE=rocksdb; @@ -45,8 +46,8 @@ hex(vb) 00000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000 -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 000000000000000001 000000000000000002 000000000000000003 @@ -111,8 +112,8 @@ hex(vb) 00000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000 -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 0000000000000000f8 0000000000000000f9 0000000000000000fa @@ -175,8 +176,8 @@ aaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 610000000000000001 616100000000000002 616161000000000003 @@ -241,8 +242,8 @@ aaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 6100000000000000f8 6161000000000000f9 6161610000000000fa diff --git a/mysql-test/suite/rocksdb/t/delete_before_lock.test b/mysql-test/suite/rocksdb/t/delete_before_lock.test index 93a9d1adaf9..52d8cff9cb8 100644 --- a/mysql-test/suite/rocksdb/t/delete_before_lock.test +++ b/mysql-test/suite/rocksdb/t/delete_before_lock.test @@ -1,6 +1,11 @@ --source include/have_rocksdb.inc --source include/have_debug_sync.inc +# The testcase doesn't work with range locking: +# UPDATE where id1=1 sets a lock on [(1;-inf), (1,+inf)] +# then DELETE will try to get a lock on (1,1) and cause lock wait timeout +--source suite/rocksdb/include/not_range_locking.inc + # This is a test case to reproduce https://github.com/facebook/mysql-5.6/issues/162 # Expected output of the last select for update was (1,2,100) and (1,3,100), but # currently it returns (1,2,1) and (1,3,1), which must be fixed. diff --git a/mysql-test/suite/rocksdb/t/range_locking.test b/mysql-test/suite/rocksdb/t/range_locking.test index 4cf6f079ebe..b0f5157f304 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.test +++ b/mysql-test/suite/rocksdb/t/range_locking.test @@ -114,10 +114,46 @@ select * from information_schema.rocksdb_locks; rollback; begin; +--echo # The following will show a range lock on 2-9 and also a point lock on 10. +--echo # This is how things currently work. select * from t1 where pk between 2 and 9 for update; select * from information_schema.rocksdb_locks; rollback; drop table t1; +--echo # +--echo # MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( + kp1 int not null, + kp2 int not null, + a int, + primary key(kp1, kp2) +) engine=rocksdb; + +insert into t1 select 1, a, 1234 from t0; +insert into t1 select 2, a, 1234 from t0; +insert into t1 select 3, a, 1234 from t0; + +connect (con1,localhost,root,,); +connection con1; + +begin; +select * from t1 where kp1=2 for update; + +connection default; +--echo # The lock on kp1=2 should inhibit the following INSERT: +--error ER_LOCK_WAIT_TIMEOUT +insert into t1 values ( 2,5,9999); +rollback; + +connection con1; +rollback; +connection default; +disconnect con1; +drop table t0,t1; diff --git a/mysql-test/suite/rocksdb/t/varbinary_format.test b/mysql-test/suite/rocksdb/t/varbinary_format.test index d00a8b7afbe..cca230614fd 100644 --- a/mysql-test/suite/rocksdb/t/varbinary_format.test +++ b/mysql-test/suite/rocksdb/t/varbinary_format.test @@ -1,6 +1,12 @@ --source include/have_debug.inc --source include/have_rocksdb.inc +# +# In point-locking mode, skip first 4 bytes, as they indexnr. +# In range-locking mode, there is one more range control byte at the front +# +set @skip_head=9 + if(@@rocksdb_use_range_locking, 2, 0); + # Create a table with a varbinary key with the current format and validate # that it sorts correctly CREATE TABLE t1( @@ -27,7 +33,7 @@ SELECT hex(vb) FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT hex(vb) FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; @@ -60,7 +66,7 @@ SELECT hex(vb) FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT hex(vb) FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; @@ -91,7 +97,7 @@ SELECT * FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT * FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; @@ -124,7 +130,7 @@ SELECT * FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT * FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; diff --git a/rocksdb b/rocksdb index 756b3e44a8b..dd648b42fde 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 756b3e44a8b7173993c0076a76c56c6a31d52a8b +Subproject commit dd648b42fde54a15675002981c305cf786db8594 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 3ec6b189f8b..72dd31bbb85 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -2275,7 +2275,9 @@ protected: virtual rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Slice &start, - const rocksdb::Slice &end) = 0; + bool start_inf_suffix, + const rocksdb::Slice &end, + bool end_inf_suffix) = 0; virtual bool prepare(const rocksdb::TransactionName &name) = 0; @@ -2729,11 +2731,33 @@ public: virtual bool is_writebatch_trx() const override { return false; } + /* + Both start and end endpoint may may be prefixes. + Both bounds are inclusive. + */ rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Slice &start, - const rocksdb::Slice &end) override + bool start_inf_suffix, + const rocksdb::Slice &end, + bool end_inf_suffix + ) override { - return m_rocksdb_tx->GetRangeLock(cf, start, end); + const char SUFFIX_INF= 0x0; + const char SUFFIX_SUP= 0x1; + // GetRangeLock accepts range endpoints, not keys. + // Convert keys to range endpoints here. + // See also range_endpoint_convert() below. + StringBuffer<64> left; + left.append(start_inf_suffix? SUFFIX_SUP :SUFFIX_INF); + left.append(start.data(), start.size()); + + StringBuffer<64> right; + right.append(end_inf_suffix? SUFFIX_SUP :SUFFIX_INF); + right.append(end.data(), end.size()); + + return m_rocksdb_tx->GetRangeLock(cf, + rocksdb::Slice(left.ptr(),left.length()), + rocksdb::Slice(right.ptr(),right.length())); } private: void release_tx(void) { @@ -3135,7 +3159,9 @@ public: rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Slice &start, - const rocksdb::Slice &end) override + bool start_inf_suffix, + const rocksdb::Slice &end, + bool end_inf_suffix) override { return rocksdb::Status::OK(); } @@ -4333,6 +4359,63 @@ void rocksdb_kill_connection(handlerton* hton, THD* thd) } +void range_endpoint_convert(const rocksdb::Slice &key, + std::string *res) +{ + const char SUFFIX_INF= 0x0; + res->clear(); + res->append(&SUFFIX_INF, 1); + res->append(key.data(), key.size()); +} + +int range_endpoints_compare(const char *a, size_t a_len, + const char *b, size_t b_len) +{ + size_t min_len= std::min(a_len, b_len); + + //compare the values. Skip the first byte as it is the endpoint signifier + int res= memcmp(a+1, b+1, min_len-1); + if (!res) + { + if (b_len > min_len) + { + // a is shorter; + if (a[0] == 0) + return -1; //"a is smaller" + else + { + // a is considered padded with 0xFF:FF:FF:FF... + return 1; // "a" is bigger + } + } + else if (a_len > min_len) + { + // the opposite of the above: b is shorter. + if (b[0] == 0) + return 1; //"b is smaller" + else + { + // b is considered padded with 0xFF:FF:FF:FF... + return -1; // "b" is bigger + } + } + else + { + // the lengths are equal (and the key values, too) + if (a[0] < b[0]) + return -1; + else if (a[0] > b[0]) + return 1; + else + return 0; + } + } + else + return res; +} + + + void Rdb_transaction::kill_lock_wait_by_thd(THD *thd) { /* @@ -4969,8 +5052,11 @@ static int rocksdb_init_func(void *const p) { if (rocksdb_use_range_locking) { - //psergey-todo: this is ugly: - rdb->get_range_lock_manager()->set_max_lock_memory(rocksdb_max_lock_memory); + rocksdb::RangeLockMgrControl *mgr= rdb->get_range_lock_manager(); + + mgr->set_endpoint_cmp_functions(range_endpoint_convert, + range_endpoints_compare); + mgr->set_max_lock_memory(rocksdb_max_lock_memory); sql_print_information("RocksDB: USING NEW RANGE LOCKING"); sql_print_information("RocksDB: Max lock memory=%lu", rocksdb_max_lock_memory); } @@ -8129,19 +8215,31 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, const rocksdb::Slice &slice, const key_range *const end_key) { - // We have 'slice' with start_slice. rocksdb::Slice end_slice; uchar end_slice_buf[MAX_KEY_LENGTH]; + bool start_has_inf_suffix, end_has_inf_suffix; if (m_lock_rows != RDB_LOCK_WRITE || !rocksdb_use_range_locking) { return; } + /* + The 'slice' parameter has the left endpoint of the range to lock. + Figure out the right endpoint + */ + if (find_flag == HA_READ_KEY_EXACT) { + /* + This is "key_part= const" interval + */ + start_has_inf_suffix= false; + end_has_inf_suffix= true; end_slice= slice; } - else - if (find_flag == HA_READ_PREFIX_LAST) { + else if (find_flag == HA_READ_PREFIX_LAST) { + //psergey-todo: swap the bounds? so that the start bounds is the end? + //Testscase! + /* We have made the kd.successor(m_sk_packed_tuple) call above. @@ -8151,8 +8249,29 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, kd.successor(end_slice_buf, slice.size()); end_slice= rocksdb::Slice((const char*)end_slice_buf, slice.size()); } - else - if (end_key) { + else if (end_key) { + // Known start range bounds: HA_READ_KEY_OR_NEXT, HA_READ_AFTER_KEY + if (find_flag == HA_READ_KEY_OR_NEXT) + start_has_inf_suffix= false; + else if (find_flag == HA_READ_AFTER_KEY) + start_has_inf_suffix= true; + else + DBUG_ASSERT(0); + + // Known end range bounds: HA_READ_AFTER_KEY, HA_READ_BEFORE_KEY + if (end_key->flag == HA_READ_AFTER_KEY) + { + // this is "key_part <= const". + end_has_inf_suffix= true; + } + else if (end_key->flag == HA_READ_BEFORE_KEY) + { + // this is "key_part < const", non-inclusive. + end_has_inf_suffix= false; + } + else + DBUG_ASSERT(0); + uchar pack_buffer[MAX_KEY_LENGTH]; uint end_slice_size= kd.pack_index_tuple(table, pack_buffer, end_slice_buf, @@ -8163,18 +8282,22 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, } else { - /* - On range scan without any end key condition, there is no - eq cond, and eq cond length is the same as index_id size (4 bytes). - Example1: id1 BIGINT, id2 INT, id3 BIGINT, PRIMARY KEY (id1, id2, id3) - WHERE id1>=1 AND id2 >= 2 and id2 <= 5 => eq_cond_len= 4 - */ + // Known start range bounds: HA_READ_KEY_OR_NEXT, HA_READ_AFTER_KEY + if (find_flag == HA_READ_KEY_OR_NEXT) + start_has_inf_suffix= false; + else if (find_flag == HA_READ_AFTER_KEY) + start_has_inf_suffix= true; + else + DBUG_ASSERT(0); + uint end_slice_size; - // TODO: or this should be kd.get_index_number() ? kd.get_infimum_key(end_slice_buf, &end_slice_size); end_slice= rocksdb::Slice((char*)end_slice_buf, end_slice_size); + end_has_inf_suffix= true; } - tx->lock_range(kd.get_cf(), slice, end_slice); + tx->lock_range(kd.get_cf(), + slice, start_has_inf_suffix, + end_slice, end_has_inf_suffix); } /* @@ -8296,11 +8419,12 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, Rdb_transaction *const tx = get_or_create_tx(table->in_use); const bool is_new_snapshot = !tx->has_snapshot(); + + set_range_lock(tx, kd, find_flag, slice, end_key); + // Loop as long as we get a deadlock error AND we end up creating the // snapshot here (i.e. it did not exist prior to this) for (;;) { - //psergey: - set_range_lock(tx, kd, find_flag, slice, end_key); /* This will open the iterator and position it at a record that's equal or diff --git a/storage/rocksdb/range_locking/ft/comparator.h b/storage/rocksdb/range_locking/ft/comparator.h index 9775cc13387..8d7fe6a0f90 100644 --- a/storage/rocksdb/range_locking/ft/comparator.h +++ b/storage/rocksdb/range_locking/ft/comparator.h @@ -44,7 +44,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "util/dbt.h" -typedef int (*ft_compare_func)(DB *db, const DBT *a, const DBT *b); +typedef int (*ft_compare_func)(DB *db, void *arg, const DBT *a, const DBT *b); int toku_keycompare(const void *key1, uint32_t key1len, const void *key2, uint32_t key2len); @@ -57,8 +57,9 @@ namespace toku { // that points may be positive or negative infinity. class comparator { - void init(ft_compare_func cmp, DESCRIPTOR desc, uint8_t memcmp_magic) { + void init(ft_compare_func cmp, void *cmp_arg, DESCRIPTOR desc, uint8_t memcmp_magic) { _cmp = cmp; + _cmp_arg= cmp_arg; _fake_db->cmp_descriptor = desc; _memcmp_magic = memcmp_magic; } @@ -67,9 +68,9 @@ namespace toku { // This magic value is reserved to mean that the magic has not been set. static const uint8_t MEMCMP_MAGIC_NONE = 0; - void create(ft_compare_func cmp, DESCRIPTOR desc, uint8_t memcmp_magic = MEMCMP_MAGIC_NONE) { + void create(ft_compare_func cmp, void *cmp_arg, DESCRIPTOR desc, uint8_t memcmp_magic = MEMCMP_MAGIC_NONE) { XCALLOC(_fake_db); - init(cmp, desc, memcmp_magic); + init(cmp, cmp_arg, desc, memcmp_magic); } // inherit the attributes of another comparator, but keep our own @@ -78,7 +79,7 @@ namespace toku { invariant_notnull(_fake_db); invariant_notnull(cmp._cmp); invariant_notnull(cmp._fake_db); - init(cmp._cmp, cmp._fake_db->cmp_descriptor, cmp._memcmp_magic); + init(cmp._cmp, cmp._cmp_arg, cmp._fake_db->cmp_descriptor, cmp._memcmp_magic); } // like inherit, but doesn't require that the this comparator @@ -120,16 +121,19 @@ namespace toku { && dbt_has_memcmp_magic(a) // ..then we expect `b' to also have the memcmp magic && __builtin_expect(dbt_has_memcmp_magic(b), 1)) { + assert(0); // psergey: this branch should not be taken. return toku_builtin_compare_fun(nullptr, a, b); } else { // yikes, const sadness here - return _cmp(const_cast<DB *>(_fake_db), a, b); + return _cmp(const_cast<DB *>(_fake_db), _cmp_arg, a, b); } } private: DB *_fake_db; ft_compare_func _cmp; + void *_cmp_arg; + uint8_t _memcmp_magic; };

1 0

[Commits] 8bdfd583ec4: Range Locking: use range endpoints, not rowkeys of range bounds
by Sergei Petrunia 04 Jan '19

04 Jan '19

revision-id: 8bdfd583ec47522874a5f3b95b91e1f7e4323a97 (fb-prod201801-180-g8bdfd583ec4) parent(s): bb04a774561548d159576b3bcaa2f638ffd76829 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-01-04 18:37:29 +0300 message: Range Locking: use range endpoints, not rowkeys of range bounds This provides correct range locking for multi-part keys. --- mysql-test/suite/rocksdb/r/range_locking.result | 50 ++++++- mysql-test/suite/rocksdb/r/varbinary_format.result | 17 ++- mysql-test/suite/rocksdb/t/delete_before_lock.test | 5 + mysql-test/suite/rocksdb/t/range_locking.test | 36 +++++ mysql-test/suite/rocksdb/t/varbinary_format.test | 14 +- rocksdb | 2 +- storage/rocksdb/ha_rocksdb.cc | 166 ++++++++++++++++++--- storage/rocksdb/range_locking/ft/comparator.h | 16 +- 8 files changed, 262 insertions(+), 44 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index 964458b783f..c3271db2730 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -90,18 +90,60 @@ pk a 10 10 select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 000001078000000a X +0 15 00000001078000000a X delete from t1 where pk between 25 and 40; select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 0000010780000019 - 0000010780000028 X -0 15 000001078000000a X +0 15 000000010780000019 - 010000010780000028 X +0 15 00000001078000000a X rollback; begin; +# The following will show a range lock on 2-9 and also a point lock on 10. +# This is how things currently work. select * from t1 where pk between 2 and 9 for update; pk a select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 16 0000010780000002 - 0000010780000009 X +0 16 00000001078000000a X +0 16 000000010780000002 - 010000010780000009 X rollback; drop table t1; +# +# MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( +kp1 int not null, +kp2 int not null, +a int, +primary key(kp1, kp2) +) engine=rocksdb; +insert into t1 select 1, a, 1234 from t0; +insert into t1 select 2, a, 1234 from t0; +insert into t1 select 3, a, 1234 from t0; +connect con1,localhost,root,,; +connection con1; +begin; +select * from t1 where kp1=2 for update; +kp1 kp2 a +2 0 1234 +2 1 1234 +2 2 1234 +2 3 1234 +2 4 1234 +2 5 1234 +2 6 1234 +2 7 1234 +2 8 1234 +2 9 1234 +connection default; +# The lock on kp1=2 should inhibit the following INSERT: +insert into t1 values ( 2,5,9999); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY +rollback; +connection con1; +rollback; +connection default; +disconnect con1; +drop table t0,t1; diff --git a/mysql-test/suite/rocksdb/r/varbinary_format.result b/mysql-test/suite/rocksdb/r/varbinary_format.result index 9362d42515c..e3dde737f9f 100644 --- a/mysql-test/suite/rocksdb/r/varbinary_format.result +++ b/mysql-test/suite/rocksdb/r/varbinary_format.result @@ -1,3 +1,4 @@ +set @skip_head=9 + if(@@rocksdb_use_range_locking, 2, 0); CREATE TABLE t1( vb VARBINARY(64) primary key ) ENGINE=rocksdb; @@ -45,8 +46,8 @@ hex(vb) 00000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000 -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 000000000000000001 000000000000000002 000000000000000003 @@ -111,8 +112,8 @@ hex(vb) 00000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000 -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 0000000000000000f8 0000000000000000f9 0000000000000000fa @@ -175,8 +176,8 @@ aaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 610000000000000001 616100000000000002 616161000000000003 @@ -241,8 +242,8 @@ aaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; -SUBSTRING(a.key,9) +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SUBSTRING(a.key,@skip_head) 6100000000000000f8 6161000000000000f9 6161610000000000fa diff --git a/mysql-test/suite/rocksdb/t/delete_before_lock.test b/mysql-test/suite/rocksdb/t/delete_before_lock.test index 93a9d1adaf9..52d8cff9cb8 100644 --- a/mysql-test/suite/rocksdb/t/delete_before_lock.test +++ b/mysql-test/suite/rocksdb/t/delete_before_lock.test @@ -1,6 +1,11 @@ --source include/have_rocksdb.inc --source include/have_debug_sync.inc +# The testcase doesn't work with range locking: +# UPDATE where id1=1 sets a lock on [(1;-inf), (1,+inf)] +# then DELETE will try to get a lock on (1,1) and cause lock wait timeout +--source suite/rocksdb/include/not_range_locking.inc + # This is a test case to reproduce https://github.com/facebook/mysql-5.6/issues/162 # Expected output of the last select for update was (1,2,100) and (1,3,100), but # currently it returns (1,2,1) and (1,3,1), which must be fixed. diff --git a/mysql-test/suite/rocksdb/t/range_locking.test b/mysql-test/suite/rocksdb/t/range_locking.test index 4cf6f079ebe..b0f5157f304 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.test +++ b/mysql-test/suite/rocksdb/t/range_locking.test @@ -114,10 +114,46 @@ select * from information_schema.rocksdb_locks; rollback; begin; +--echo # The following will show a range lock on 2-9 and also a point lock on 10. +--echo # This is how things currently work. select * from t1 where pk between 2 and 9 for update; select * from information_schema.rocksdb_locks; rollback; drop table t1; +--echo # +--echo # MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( + kp1 int not null, + kp2 int not null, + a int, + primary key(kp1, kp2) +) engine=rocksdb; + +insert into t1 select 1, a, 1234 from t0; +insert into t1 select 2, a, 1234 from t0; +insert into t1 select 3, a, 1234 from t0; + +connect (con1,localhost,root,,); +connection con1; + +begin; +select * from t1 where kp1=2 for update; + +connection default; +--echo # The lock on kp1=2 should inhibit the following INSERT: +--error ER_LOCK_WAIT_TIMEOUT +insert into t1 values ( 2,5,9999); +rollback; + +connection con1; +rollback; +connection default; +disconnect con1; +drop table t0,t1; diff --git a/mysql-test/suite/rocksdb/t/varbinary_format.test b/mysql-test/suite/rocksdb/t/varbinary_format.test index d00a8b7afbe..cca230614fd 100644 --- a/mysql-test/suite/rocksdb/t/varbinary_format.test +++ b/mysql-test/suite/rocksdb/t/varbinary_format.test @@ -1,6 +1,12 @@ --source include/have_debug.inc --source include/have_rocksdb.inc +# +# In point-locking mode, skip first 4 bytes, as they indexnr. +# In range-locking mode, there is one more range control byte at the front +# +set @skip_head=9 + if(@@rocksdb_use_range_locking, 2, 0); + # Create a table with a varbinary key with the current format and validate # that it sorts correctly CREATE TABLE t1( @@ -27,7 +33,7 @@ SELECT hex(vb) FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT hex(vb) FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; @@ -60,7 +66,7 @@ SELECT hex(vb) FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT hex(vb) FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; @@ -91,7 +97,7 @@ SELECT * FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT * FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; @@ -124,7 +130,7 @@ SELECT * FROM t1; # validate that the keys were encoded as expected BEGIN; SELECT * FROM t1 FOR UPDATE; -SELECT SUBSTRING(a.key,9) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; +SELECT SUBSTRING(a.key,@skip_head) FROM information_schema.rocksdb_locks AS a ORDER BY a.key; ROLLBACK; DROP TABLE t1; diff --git a/rocksdb b/rocksdb index 756b3e44a8b..dd648b42fde 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 756b3e44a8b7173993c0076a76c56c6a31d52a8b +Subproject commit dd648b42fde54a15675002981c305cf786db8594 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 3ec6b189f8b..72dd31bbb85 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -2275,7 +2275,9 @@ protected: virtual rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Slice &start, - const rocksdb::Slice &end) = 0; + bool start_inf_suffix, + const rocksdb::Slice &end, + bool end_inf_suffix) = 0; virtual bool prepare(const rocksdb::TransactionName &name) = 0; @@ -2729,11 +2731,33 @@ public: virtual bool is_writebatch_trx() const override { return false; } + /* + Both start and end endpoint may may be prefixes. + Both bounds are inclusive. + */ rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Slice &start, - const rocksdb::Slice &end) override + bool start_inf_suffix, + const rocksdb::Slice &end, + bool end_inf_suffix + ) override { - return m_rocksdb_tx->GetRangeLock(cf, start, end); + const char SUFFIX_INF= 0x0; + const char SUFFIX_SUP= 0x1; + // GetRangeLock accepts range endpoints, not keys. + // Convert keys to range endpoints here. + // See also range_endpoint_convert() below. + StringBuffer<64> left; + left.append(start_inf_suffix? SUFFIX_SUP :SUFFIX_INF); + left.append(start.data(), start.size()); + + StringBuffer<64> right; + right.append(end_inf_suffix? SUFFIX_SUP :SUFFIX_INF); + right.append(end.data(), end.size()); + + return m_rocksdb_tx->GetRangeLock(cf, + rocksdb::Slice(left.ptr(),left.length()), + rocksdb::Slice(right.ptr(),right.length())); } private: void release_tx(void) { @@ -3135,7 +3159,9 @@ public: rocksdb::Status lock_range(rocksdb::ColumnFamilyHandle *const cf, const rocksdb::Slice &start, - const rocksdb::Slice &end) override + bool start_inf_suffix, + const rocksdb::Slice &end, + bool end_inf_suffix) override { return rocksdb::Status::OK(); } @@ -4333,6 +4359,63 @@ void rocksdb_kill_connection(handlerton* hton, THD* thd) } +void range_endpoint_convert(const rocksdb::Slice &key, + std::string *res) +{ + const char SUFFIX_INF= 0x0; + res->clear(); + res->append(&SUFFIX_INF, 1); + res->append(key.data(), key.size()); +} + +int range_endpoints_compare(const char *a, size_t a_len, + const char *b, size_t b_len) +{ + size_t min_len= std::min(a_len, b_len); + + //compare the values. Skip the first byte as it is the endpoint signifier + int res= memcmp(a+1, b+1, min_len-1); + if (!res) + { + if (b_len > min_len) + { + // a is shorter; + if (a[0] == 0) + return -1; //"a is smaller" + else + { + // a is considered padded with 0xFF:FF:FF:FF... + return 1; // "a" is bigger + } + } + else if (a_len > min_len) + { + // the opposite of the above: b is shorter. + if (b[0] == 0) + return 1; //"b is smaller" + else + { + // b is considered padded with 0xFF:FF:FF:FF... + return -1; // "b" is bigger + } + } + else + { + // the lengths are equal (and the key values, too) + if (a[0] < b[0]) + return -1; + else if (a[0] > b[0]) + return 1; + else + return 0; + } + } + else + return res; +} + + + void Rdb_transaction::kill_lock_wait_by_thd(THD *thd) { /* @@ -4969,8 +5052,11 @@ static int rocksdb_init_func(void *const p) { if (rocksdb_use_range_locking) { - //psergey-todo: this is ugly: - rdb->get_range_lock_manager()->set_max_lock_memory(rocksdb_max_lock_memory); + rocksdb::RangeLockMgrControl *mgr= rdb->get_range_lock_manager(); + + mgr->set_endpoint_cmp_functions(range_endpoint_convert, + range_endpoints_compare); + mgr->set_max_lock_memory(rocksdb_max_lock_memory); sql_print_information("RocksDB: USING NEW RANGE LOCKING"); sql_print_information("RocksDB: Max lock memory=%lu", rocksdb_max_lock_memory); } @@ -8129,19 +8215,31 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, const rocksdb::Slice &slice, const key_range *const end_key) { - // We have 'slice' with start_slice. rocksdb::Slice end_slice; uchar end_slice_buf[MAX_KEY_LENGTH]; + bool start_has_inf_suffix, end_has_inf_suffix; if (m_lock_rows != RDB_LOCK_WRITE || !rocksdb_use_range_locking) { return; } + /* + The 'slice' parameter has the left endpoint of the range to lock. + Figure out the right endpoint + */ + if (find_flag == HA_READ_KEY_EXACT) { + /* + This is "key_part= const" interval + */ + start_has_inf_suffix= false; + end_has_inf_suffix= true; end_slice= slice; } - else - if (find_flag == HA_READ_PREFIX_LAST) { + else if (find_flag == HA_READ_PREFIX_LAST) { + //psergey-todo: swap the bounds? so that the start bounds is the end? + //Testscase! + /* We have made the kd.successor(m_sk_packed_tuple) call above. @@ -8151,8 +8249,29 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, kd.successor(end_slice_buf, slice.size()); end_slice= rocksdb::Slice((const char*)end_slice_buf, slice.size()); } - else - if (end_key) { + else if (end_key) { + // Known start range bounds: HA_READ_KEY_OR_NEXT, HA_READ_AFTER_KEY + if (find_flag == HA_READ_KEY_OR_NEXT) + start_has_inf_suffix= false; + else if (find_flag == HA_READ_AFTER_KEY) + start_has_inf_suffix= true; + else + DBUG_ASSERT(0); + + // Known end range bounds: HA_READ_AFTER_KEY, HA_READ_BEFORE_KEY + if (end_key->flag == HA_READ_AFTER_KEY) + { + // this is "key_part <= const". + end_has_inf_suffix= true; + } + else if (end_key->flag == HA_READ_BEFORE_KEY) + { + // this is "key_part < const", non-inclusive. + end_has_inf_suffix= false; + } + else + DBUG_ASSERT(0); + uchar pack_buffer[MAX_KEY_LENGTH]; uint end_slice_size= kd.pack_index_tuple(table, pack_buffer, end_slice_buf, @@ -8163,18 +8282,22 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, } else { - /* - On range scan without any end key condition, there is no - eq cond, and eq cond length is the same as index_id size (4 bytes). - Example1: id1 BIGINT, id2 INT, id3 BIGINT, PRIMARY KEY (id1, id2, id3) - WHERE id1>=1 AND id2 >= 2 and id2 <= 5 => eq_cond_len= 4 - */ + // Known start range bounds: HA_READ_KEY_OR_NEXT, HA_READ_AFTER_KEY + if (find_flag == HA_READ_KEY_OR_NEXT) + start_has_inf_suffix= false; + else if (find_flag == HA_READ_AFTER_KEY) + start_has_inf_suffix= true; + else + DBUG_ASSERT(0); + uint end_slice_size; - // TODO: or this should be kd.get_index_number() ? kd.get_infimum_key(end_slice_buf, &end_slice_size); end_slice= rocksdb::Slice((char*)end_slice_buf, end_slice_size); + end_has_inf_suffix= true; } - tx->lock_range(kd.get_cf(), slice, end_slice); + tx->lock_range(kd.get_cf(), + slice, start_has_inf_suffix, + end_slice, end_has_inf_suffix); } /* @@ -8296,11 +8419,12 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, Rdb_transaction *const tx = get_or_create_tx(table->in_use); const bool is_new_snapshot = !tx->has_snapshot(); + + set_range_lock(tx, kd, find_flag, slice, end_key); + // Loop as long as we get a deadlock error AND we end up creating the // snapshot here (i.e. it did not exist prior to this) for (;;) { - //psergey: - set_range_lock(tx, kd, find_flag, slice, end_key); /* This will open the iterator and position it at a record that's equal or diff --git a/storage/rocksdb/range_locking/ft/comparator.h b/storage/rocksdb/range_locking/ft/comparator.h index 9775cc13387..8d7fe6a0f90 100644 --- a/storage/rocksdb/range_locking/ft/comparator.h +++ b/storage/rocksdb/range_locking/ft/comparator.h @@ -44,7 +44,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "util/dbt.h" -typedef int (*ft_compare_func)(DB *db, const DBT *a, const DBT *b); +typedef int (*ft_compare_func)(DB *db, void *arg, const DBT *a, const DBT *b); int toku_keycompare(const void *key1, uint32_t key1len, const void *key2, uint32_t key2len); @@ -57,8 +57,9 @@ namespace toku { // that points may be positive or negative infinity. class comparator { - void init(ft_compare_func cmp, DESCRIPTOR desc, uint8_t memcmp_magic) { + void init(ft_compare_func cmp, void *cmp_arg, DESCRIPTOR desc, uint8_t memcmp_magic) { _cmp = cmp; + _cmp_arg= cmp_arg; _fake_db->cmp_descriptor = desc; _memcmp_magic = memcmp_magic; } @@ -67,9 +68,9 @@ namespace toku { // This magic value is reserved to mean that the magic has not been set. static const uint8_t MEMCMP_MAGIC_NONE = 0; - void create(ft_compare_func cmp, DESCRIPTOR desc, uint8_t memcmp_magic = MEMCMP_MAGIC_NONE) { + void create(ft_compare_func cmp, void *cmp_arg, DESCRIPTOR desc, uint8_t memcmp_magic = MEMCMP_MAGIC_NONE) { XCALLOC(_fake_db); - init(cmp, desc, memcmp_magic); + init(cmp, cmp_arg, desc, memcmp_magic); } // inherit the attributes of another comparator, but keep our own @@ -78,7 +79,7 @@ namespace toku { invariant_notnull(_fake_db); invariant_notnull(cmp._cmp); invariant_notnull(cmp._fake_db); - init(cmp._cmp, cmp._fake_db->cmp_descriptor, cmp._memcmp_magic); + init(cmp._cmp, cmp._cmp_arg, cmp._fake_db->cmp_descriptor, cmp._memcmp_magic); } // like inherit, but doesn't require that the this comparator @@ -120,16 +121,19 @@ namespace toku { && dbt_has_memcmp_magic(a) // ..then we expect `b' to also have the memcmp magic && __builtin_expect(dbt_has_memcmp_magic(b), 1)) { + assert(0); // psergey: this branch should not be taken. return toku_builtin_compare_fun(nullptr, a, b); } else { // yikes, const sadness here - return _cmp(const_cast<DB *>(_fake_db), a, b); + return _cmp(const_cast<DB *>(_fake_db), _cmp_arg, a, b); } } private: DB *_fake_db; ft_compare_func _cmp; + void *_cmp_arg; + uint8_t _memcmp_magic; };

1 0

[Commits] dd648b42f: Range locking: switch to using end-points, not rowkeys.
by Sergei Petrunia 04 Jan '19

04 Jan '19

revision-id: dd648b42fde54a15675002981c305cf786db8594 (v5.8-1023-gdd648b42f) parent(s): 756b3e44a8b7173993c0076a76c56c6a31d52a8b author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-01-04 14:18:04 +0300 message: Range locking: switch to using end-points, not rowkeys. In addition to row comparison function, the user will provide a endpoint-comparison function, and a function to convert rowkey to endpoint. --- include/rocksdb/utilities/transaction_db.h | 12 +++++++- utilities/transactions/transaction_lock_mgr.cc | 41 ++++++++++++++++++++++++-- utilities/transactions/transaction_lock_mgr.h | 26 ++++++++++------ 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/include/rocksdb/utilities/transaction_db.h b/include/rocksdb/utilities/transaction_db.h index 16fa8a4a7..aa68a6243 100644 --- a/include/rocksdb/utilities/transaction_db.h +++ b/include/rocksdb/utilities/transaction_db.h @@ -195,7 +195,17 @@ struct DeadlockPath { // Interface for controlling Range Locking manager class RangeLockMgrControl { -public: + public: + + typedef void (*convert_key_to_endpoint_func)(const rocksdb::Slice &key, + std::string *endpoint); + + typedef int (*compare_endpoints_func)(const char *a, size_t a_len, + const char *b, size_t b_len); + + virtual void set_endpoint_cmp_functions(convert_key_to_endpoint_func cvt_func, + compare_endpoints_func cmp_func)=0; + virtual int set_max_lock_memory(size_t max_lock_memory) = 0; virtual uint64_t get_escalation_count() = 0; }; diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index 561531f7e..53e60679d 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -449,7 +449,10 @@ Status RangeLockMgr::TryLock(PessimisticTransaction* txn, uint32_t column_family_id, const std::string& key, Env* env, bool exclusive) { - return TryRangeLock(txn, column_family_id, key, key, exclusive); + std::string endpoint; + convert_key_to_endpoint(rocksdb::Slice(key.data(), key.size()), &endpoint); + rocksdb::Slice endp_slice(endpoint.data(), endpoint.length()); + return TryRangeLock(txn, column_family_id, endp_slice, endp_slice, exclusive); } @@ -854,8 +857,9 @@ void RangeLockMgr::UnLockAll(const PessimisticTransaction* txn, if (txn->owned_locks) { RangeLockList* range_list= (RangeLockList*)txn->owned_locks.get(); - lt->release_locks(txn->GetID(), &range_list->buffer); + lt->release_locks(txn->GetID(), &range_list->buffer, true); range_list->buffer.destroy(); + range_list->buffer.create(); toku::lock_request::retry_all_lock_requests(lt, nullptr /* lock_wait_needed_callback */); } @@ -877,6 +881,39 @@ void RangeLockMgr::UnLockAll(const PessimisticTransaction* txn, #endif } +int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, + const DBT *a_key, + const DBT *b_key) +{ + RangeLockMgr* mgr= (RangeLockMgr*) arg; + return mgr->compare_endpoints((const char*)a_key->data, a_key->size, + (const char*)b_key->data, b_key->size); +} + + +RangeLockMgr::RangeLockMgr(TransactionDB* txn_db) : my_txn_db(txn_db) +{ + ltm.create(on_create, on_destroy, on_escalate, NULL); + lt= nullptr; +} + +void RangeLockMgr::set_endpoint_cmp_functions(convert_key_to_endpoint_func cvt_func, + compare_endpoints_func cmp_func) +{ + convert_key_to_endpoint= cvt_func; + compare_endpoints= cmp_func; + + // The rest is like a constructor: + assert(!lt); + + toku::comparator cmp; + //cmp.create(toku_builtin_compare_fun, NULL); + cmp.create(compare_dbt_endpoints, (void*)this, NULL); + DICTIONARY_ID dict_id = { .dictid = 1 }; + lt= ltm.get_lt(dict_id, cmp , /* on_create_extra*/nullptr); +} + + uint64_t RangeLockMgr::get_escalation_count() { LTM_STATUS_S ltm_status_test; diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index 9f7f61db7..59234c26f 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -217,15 +217,7 @@ class RangeLockMgr : void UnLock(PessimisticTransaction* txn, uint32_t column_family_id, const std::string& key, Env* env) override ; - RangeLockMgr(TransactionDB* txn_db) : my_txn_db(txn_db) - { - ltm.create(on_create, on_destroy, on_escalate, NULL); - toku::comparator cmp; - cmp.create(toku_builtin_compare_fun, NULL); - DICTIONARY_ID dict_id = { .dictid = 1 }; - lt= ltm.get_lt(dict_id, cmp , /* on_create_extra*/nullptr); - - } + RangeLockMgr(TransactionDB* txn_db); void KillLockWait(void *cdata); @@ -238,11 +230,27 @@ class RangeLockMgr : TransactionLockMgr::LockStatusData GetLockStatusData(); + using RangeLockMgrControl::convert_key_to_endpoint_func; + using RangeLockMgrControl::compare_endpoints_func; + + void set_endpoint_cmp_functions(convert_key_to_endpoint_func cvt_func, + compare_endpoints_func cmp_func) override; + private: toku::locktree_manager ltm; toku::locktree *lt; // only one tree for now + // Convert rowkey to endpoint (TODO: shouldn't "rowkey=const" translate into + // a pair of [start; end] endpoints in general? They translate into the same + // value in our current encoding, but...) + convert_key_to_endpoint_func convert_key_to_endpoint; + + // User-provided endpoint comparison function + compare_endpoints_func compare_endpoints; + TransactionDB* my_txn_db; + + static int compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); // Callbacks static int on_create(locktree *lt, void *extra) { return 0; /* no error */ }

1 0

[Commits] bb04a774561: Disable rocksdb.unique_check in range-locking mode due to MDEV-17873
by Sergei Petrunia 03 Jan '19

03 Jan '19

revision-id: bb04a774561548d159576b3bcaa2f638ffd76829 (fb-prod201801-179-gbb04a774561) parent(s): 5ebf1134635d886cbadce7665830831cdddbafc3 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-01-03 22:28:40 +0300 message: Disable rocksdb.unique_check in range-locking mode due to MDEV-17873 --- mysql-test/suite/rocksdb/t/unique_check.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql-test/suite/rocksdb/t/unique_check.test b/mysql-test/suite/rocksdb/t/unique_check.test index 86ae15924cb..390f590dc4b 100644 --- a/mysql-test/suite/rocksdb/t/unique_check.test +++ b/mysql-test/suite/rocksdb/t/unique_check.test @@ -1,6 +1,11 @@ --source include/have_rocksdb.inc --source include/have_debug_sync.inc +# Doesn't work with range locking because lock tree waits do not set +# state="Waiting for row lock" in I_S.PROCESSLIST. See MDEV-17873 for +# details. +--source suite/rocksdb/include/not_range_locking.inc + # For GitHub issue#167 -- Unique key check doesn't work connect (con1, localhost, root,,);

1 0

[Commits] d129890e20d: MDEV-18117: Crash with Explain extended when using limit rows examined
by Varun 02 Jan '19

02 Jan '19

revision-id: d129890e20d53bff1ff73391dacc0b2ad697d0e1 (mariadb-10.4.0-218-gd129890e20d) parent(s): 2465d3e00b2f18c7d4967f9bb10438883682787d author: Varun Gupta committer: Varun Gupta timestamp: 2019-01-02 23:08:01 +0530 message: MDEV-18117: Crash with Explain extended when using limit rows examined No need to set SELECT_LEX:explicit_limit when we have a limit clause that sets only the EXAMINED ROWS. --- mysql-test/main/limit_rows_examined.result | 15 +++++++++++++++ mysql-test/main/limit_rows_examined.test | 12 ++++++++++++ sql/sql_yacc.yy | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/limit_rows_examined.result b/mysql-test/main/limit_rows_examined.result index 71b99d066ff..0711e10ddde 100644 --- a/mysql-test/main/limit_rows_examined.result +++ b/mysql-test/main/limit_rows_examined.result @@ -864,3 +864,18 @@ Warnings: Warning 1931 Query execution was interrupted. The query examined at least 22 rows, which exceeds LIMIT ROWS EXAMINED (21). The query result may be incomplete drop view v; drop table t1, t2; +# +# MDEV-18117: Crash with Explain extended when using limit rows examined +# +create table t1 (c1 char(2)); +create table t2 (c2 char(2)); +insert into t1 values ('bb'), ('cc'), ('aa'), ('dd'); +insert into t2 values ('bb'), ('cc'), ('dd'), ('ff'); +explain extended +select * from t1, t2 where c1 = c2 LIMIT ROWS EXAMINED 2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 100.00 +1 SIMPLE t2 ALL NULL NULL NULL NULL 4 100.00 Using where; Using join buffer (flat, BNL join) +Warnings: +Note 1003 select `test`.`t1`.`c1` AS `c1`,`test`.`t2`.`c2` AS `c2` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`c2` = `test`.`t1`.`c1` +drop table t1,t2; diff --git a/mysql-test/main/limit_rows_examined.test b/mysql-test/main/limit_rows_examined.test index 815394aec5c..528bb8c7b3d 100644 --- a/mysql-test/main/limit_rows_examined.test +++ b/mysql-test/main/limit_rows_examined.test @@ -576,3 +576,15 @@ EXECUTE ps; drop view v; drop table t1, t2; + +--echo # +--echo # MDEV-18117: Crash with Explain extended when using limit rows examined +--echo # + +create table t1 (c1 char(2)); +create table t2 (c2 char(2)); +insert into t1 values ('bb'), ('cc'), ('aa'), ('dd'); +insert into t2 values ('bb'), ('cc'), ('dd'), ('ff'); +explain extended +select * from t1, t2 where c1 = c2 LIMIT ROWS EXAMINED 2; +drop table t1,t2; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index f628ef098a1..ad51397ee71 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -12633,7 +12633,7 @@ limit_clause: { $$.select_limit= 0; $$.offset_limit= 0; - $$.explicit_limit= 1; + $$.explicit_limit= 0; Lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_LIMIT); } ;

1 0

[Commits] 838dcdd551a: Merge remote-tracking branch 'origin/10.4' into bb-10.4-galera4
by jan 02 Jan '19

02 Jan '19

revision-id: 838dcdd551afdb62abdcf4abe122fc00d12331e8 (mariadb-10.3.6-279-g838dcdd551a) parent(s): a237499f178a61782f6bd5ec31835b367c35de91 2465d3e00b2f18c7d4967f9bb10438883682787d author: Jan Lindström committer: Jan Lindström timestamp: 2019-01-02 08:39:01 +0200 message: Merge remote-tracking branch 'origin/10.4' into bb-10.4-galera4 include/my_counter.h | 3 ++ plugin/query_response_time/query_response_time.cc | 39 ++++++++++------------- sql/log_event.cc | 10 ++---- sql/mysqld.h | 1 + sql/opt_range.cc | 4 --- sql/rpl_mi.h | 6 ++-- sql/slave.cc | 13 ++------ sql/sql_class.cc | 31 +++++++++--------- sql/sql_statistics.cc | 1 - sql/table.cc | 1 - sql/table_cache.cc | 6 ++-- storage/innobase/buf/buf0buf.cc | 4 +-- storage/innobase/dict/dict0dict.cc | 13 ++++---- storage/innobase/fil/fil0fil.cc | 2 +- storage/innobase/gis/gis0sea.cc | 2 +- storage/innobase/ha/ha0ha.cc | 20 ++++-------- storage/innobase/handler/ha_innodb.cc | 18 ++++------- storage/innobase/ibuf/ibuf0ibuf.cc | 28 +++++++--------- storage/innobase/include/buf0buf.h | 12 +++---- storage/innobase/include/dict0mem.h | 3 +- storage/innobase/include/fil0fil.h | 38 ++++++---------------- storage/innobase/include/ib0mutex.h | 30 +++++++++-------- storage/innobase/include/ibuf0ibuf.ic | 4 +-- storage/innobase/include/read0types.h | 34 ++++++++++++-------- storage/innobase/include/sync0types.h | 18 ----------- storage/innobase/include/ut0ut.h | 1 - storage/innobase/read/read0read.cc | 10 +++--- storage/innobase/srv/srv0srv.cc | 25 ++++++++------- 28 files changed, 155 insertions(+), 222 deletions(-) diff --cc sql/sql_class.cc index a0d2ba8a362,2952adbd3e6..045ead31515 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@@ -66,14 -66,8 +66,13 @@@ #include "sql_callback.h" #include "lock.h" #include "wsrep_mysqld.h" -#include "wsrep_thd.h" #include "sql_connect.h" - #include "my_atomic.h" +#ifdef WITH_WSREP +#include "mysql/service_wsrep.h" +#include "wsrep_binlog.h" /* wsrep_fragment_unit() */ +#include "wsrep_thd.h" +#include "wsrep_trans_observer.h" +#endif /* WITH_WSREP */ #ifdef HAVE_SYS_SYSCALL_H #include <sys/syscall.h>

1 0