- commits - lists.mariadb.org

[Commits] 293d69024: Range Locking : minor fixes
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 293d6902411748728fb48bf63e63ba121a963788 (v5.8-1039-g293d69024) parent(s): d334cf12818bfd2d44dc06c7bbe78c3a18bca543 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 18:11:02 +0300 message: Range Locking : minor fixes - TransactionDB::init: Unused variable - RangeLockList::append: always append the new range to the list (even if we've just created the transaction) - PessimisticTransactionDB::TryRangeLock: safer value for currently-unused parameter --- utilities/transactions/pessimistic_transaction_db.cc | 2 +- utilities/transactions/transaction_lock_mgr.cc | 4 +--- utilities/transactions/transaction_lock_mgr.h | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/utilities/transactions/pessimistic_transaction_db.cc b/utilities/transactions/pessimistic_transaction_db.cc index e3ce435af..14b82de00 100644 --- a/utilities/transactions/pessimistic_transaction_db.cc +++ b/utilities/transactions/pessimistic_transaction_db.cc @@ -405,7 +405,7 @@ PessimisticTransactionDB::TryRangeLock(PessimisticTransaction *txn, const Endpoint& end_endp) { if (range_lock_mgr_) { return range_lock_mgr_->TryRangeLock(txn, cfh_id, start_endp, - end_endp, /*exclusive=*/false); + end_endp, /*exclusive=*/true); } else return Status::NotSupported(); diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index c375b24f2..95e819b3a 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -798,12 +798,10 @@ public: auto it= buffers_.find(cf_id); if (it == buffers_.end()) { // create a new one - //it->second.create(); it= buffers_.emplace(cf_id, std::shared_ptr<toku::range_buffer>(new toku::range_buffer())).first; it->second->create(); } - else - it->second->append(left_key, right_key); + it->second->append(left_key, right_key); } std::unordered_map<uint32_t, std::shared_ptr<toku::range_buffer>> buffers_; diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index d35bf5935..1f5dbc634 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -78,7 +78,7 @@ class BaseLockMgr { virtual std::vector<DeadlockPath> GetDeadlockInfoBuffer()= 0; // TransactionDB will call this at start - virtual void init(TransactionDB *db_arg) {}; + virtual void init(TransactionDB*) {}; virtual ~BaseLockMgr(){} using LockStatusData = std::unordered_multimap<uint32_t, KeyLockInfo>;

1 0

[Commits] c523d5bd0e7: Range Locking: let RocksDB use user-provided comparator
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: c523d5bd0e776a88495405dc9ba305da9fe17600 (fb-prod201801-227-gc523d5bd0e7) parent(s): 70e12615732d658939972b24da162de8d57086f0 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 16:54:38 +0300 message: Range Locking: let RocksDB use user-provided comparator The order of endpoints in GetRangeLock() call must match the used ordering --- mysql-test/suite/rocksdb/r/range_locking_rev_cf.result | 4 ++-- rocksdb | 2 +- storage/rocksdb/ha_rocksdb.cc | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result index 444c51162cd..c2f8064c6b7 100644 --- a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -97,8 +97,8 @@ COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE delete from t1 where pk between 25 and 40; select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 000000010780000019 - 010000010780000028 X 0 15 00000001078000000a X +0 15 000000010780000028 - 010000010780000019 X rollback; begin; # The following will show a range lock on 2-9 and also a point lock on 10. @@ -107,8 +107,8 @@ select * from t1 where pk between 2 and 9 for update; pk a select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE +0 16 000000010780000009 - 010000010780000002 X 0 16 00000001078000000a X -0 16 000000010780000002 - 010000010780000009 X rollback; drop table t1; # diff --git a/rocksdb b/rocksdb index 9af13f0e966..d334cf12818 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 9af13f0e966171ef8688484b2da7fa576a1b4574 +Subproject commit d334cf12818bfd2d44dc06c7bbe78c3a18bca543 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index dea7fb9e53f..0c4dea535ca 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8245,9 +8245,18 @@ int ha_rocksdb::set_range_lock(Rdb_transaction *tx, end_has_inf_suffix= true; } - auto s= tx->lock_range(kd.get_cf(), - rocksdb::Endpoint(slice, start_has_inf_suffix), - rocksdb::Endpoint(end_slice, end_has_inf_suffix)); + rocksdb::Endpoint start_endp; + rocksdb::Endpoint end_endp; + if (kd.m_is_reverse_cf) { + // Flip the endpoints + start_endp =rocksdb::Endpoint(end_slice, !end_has_inf_suffix); + end_endp = rocksdb::Endpoint(slice, !start_has_inf_suffix); + } else { + start_endp= rocksdb::Endpoint(slice, start_has_inf_suffix); + end_endp= rocksdb::Endpoint(end_slice, end_has_inf_suffix); + } + + auto s= tx->lock_range(kd.get_cf(), start_endp, end_endp); if (!s.ok()) { return (tx->set_status_error(table->in_use, s, kd, m_tbl_def, m_table_handler));

1 0

[Commits] d334cf128: Range Locking: make the range tree code invoke the user-provided comparator
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: d334cf12818bfd2d44dc06c7bbe78c3a18bca543 (v5.8-1038-gd334cf128) parent(s): 9af13f0e966171ef8688484b2da7fa576a1b4574 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 16:53:06 +0300 message: Range Locking: make the range tree code invoke the user-provided comparator --- include/rocksdb/utilities/transaction.h | 8 +++++ utilities/transactions/transaction_lock_mgr.cc | 49 ++++++++++---------------- utilities/transactions/transaction_lock_mgr.h | 8 +---- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/include/rocksdb/utilities/transaction.h b/include/rocksdb/utilities/transaction.h index 20f9b1bbd..b0f1308ed 100644 --- a/include/rocksdb/utilities/transaction.h +++ b/include/rocksdb/utilities/transaction.h @@ -80,6 +80,12 @@ using TransactionID = uint64_t; class Endpoint { public: Slice slice; + + /* + true : the key has an "infinity" suffix. A suffix that would compare as + greater than any other suffix + false : otherwise + */ bool inf_suffix; Endpoint(const Slice &slice_arg, bool inf_suffix_arg=false) : @@ -90,6 +96,8 @@ class Endpoint { Endpoint(const char* s, size_t size, bool inf_suffix_arg=false) : slice(s, size), inf_suffix(inf_suffix_arg) {} + + Endpoint() : inf_suffix(false) {} }; // Provides notification to the caller of SetSnapshotOnNextOperation when diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index c98803acb..c375b24f2 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -813,11 +813,12 @@ public: bool releasing_locks_; }; +static const char SUFFIX_INFIMUM= 0x0; +static const char SUFFIX_SUPREMUM= 0x1; + static void serialize_endpoint(const Endpoint &endp, std::string *buf) { - const char SUFFIX_INF= 0x0; - const char SUFFIX_SUP= 0x1; - buf->push_back(endp.inf_suffix ? SUFFIX_SUP : SUFFIX_INF); + buf->push_back(endp.inf_suffix ? SUFFIX_SUPREMUM : SUFFIX_INFIMUM); buf->append(endp.slice.data(), endp.slice.size()); } @@ -1044,11 +1045,6 @@ void RangeLockMgr::UnLockAll(const PessimisticTransaction* txn, Env*) { int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key) { - //RangeLockMgr* mgr= (RangeLockMgr*) arg; - // TODO: this should compare endpoints using the user-provided comparator + - // endpoint encoding. - // (just use one from any column family) - const char *a= (const char*)a_key->data; const char *b= (const char*)b_key->data; @@ -1057,16 +1053,16 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, size_t min_len= std::min(a_len, b_len); - //compare the values. Skip the first byte as it is the endpoint signifier - - //TODO: use the upper layer' comparison function. - int res= memcmp(a+1, b+1, min_len-1); + // Compare the values. The first byte encodes the endpoint type, its value + // is either SUFFIX_INFIMUM or SUFFIX_SUPREMUM. + Comparator *cmp = (Comparator*) arg; + int res= cmp->Compare(Slice(a+1, min_len-1), Slice(b+1, min_len-1)); if (!res) { if (b_len > min_len) { // a is shorter; - if (a[0] == 0) + if (a[0] == SUFFIX_INFIMUM) return -1; //"a is smaller" else { @@ -1077,7 +1073,7 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, else if (a_len > min_len) { // the opposite of the above: b is shorter. - if (b[0] == 0) + if (b[0] == SUFFIX_INFIMUM) return 1; //"b is smaller" else { @@ -1100,21 +1096,10 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, return res; } -int RangeLockMgr::compare_dbt_endpoints_rev(__toku_db* db, void *arg, - const DBT *a_key, - const DBT *b_key) { - return -compare_dbt_endpoints(db, arg, a_key, b_key); -} - - RangeLockMgr::RangeLockMgr(std::shared_ptr<TransactionDBMutexFactory> mutex_factory) : mutex_factory_(mutex_factory), ltree_lookup_cache_(new ThreadLocalPtr(nullptr)) { - ltm_.create(on_create, on_destroy, on_escalate, NULL, mutex_factory_); - - fw_cmp_.create(compare_dbt_endpoints, (void*)this, NULL); - bw_cmp_.create(compare_dbt_endpoints, (void*)this, NULL); } @@ -1168,8 +1153,6 @@ RangeLockMgr::~RangeLockMgr() { ltm_.release_lt(it.second); } ltm_.destroy(); - fw_cmp_.destroy(); - bw_cmp_.destroy(); } uint64_t RangeLockMgr::get_escalation_count() { @@ -1199,18 +1182,24 @@ void RangeLockMgr::AddColumnFamily(const ColumnFamilyHandle *cfh) { InstrumentedMutexLock l(&ltree_map_mutex_); if (ltree_map_.find(column_family_id) == ltree_map_.end()) { DICTIONARY_ID dict_id = { .dictid = column_family_id }; - toku::comparator *ltree_cmp; // "RocksDB_SE_v3.10" // BytewiseComparator() ,ReverseBytewiseComparator() + /* + toku::comparator *ltree_cmp; if (!strcmp(cfh->GetComparator()->Name(), "RocksDB_SE_v3.10")) ltree_cmp = &fw_cmp_; else if (!strcmp(cfh->GetComparator()->Name(),"rev:RocksDB_SE_v3.10")) - ltree_cmp = &fw_cmp_; // temporary: use the same ordering + ltree_cmp = &bw_cmp_; else { assert(false); ltree_cmp= nullptr; } - toku::locktree *ltree= ltm_.get_lt(dict_id, *ltree_cmp, + */ + toku::comparator cmp; + cmp.create(compare_dbt_endpoints, (void*)cfh->GetComparator(), NULL); + toku::locktree *ltree= ltm_.get_lt(dict_id, cmp, /* on_create_extra*/nullptr); + // This is ok to because get_lt has copied the comparator: + cmp.destroy(); ltree_map_.emplace(column_family_id, ltree); } else { // column_family already exists in lock map diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index e41a4961a..d35bf5935 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -250,9 +250,6 @@ class RangeLockMgr : private: toku::locktree_manager ltm_; - toku::comparator fw_cmp_; - toku::comparator bw_cmp_; - TransactionDB* my_txn_db_; std::shared_ptr<TransactionDBMutexFactory> mutex_factory_; @@ -269,8 +266,7 @@ class RangeLockMgr : toku::locktree *get_locktree_by_cfid(uint32_t cf_id); static int compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); - static int compare_dbt_endpoints_rev(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); - + // Callbacks static int on_create(locktree*, void*) { return 0; /* no error */ } static void on_destroy(locktree*) {} @@ -279,7 +275,5 @@ class RangeLockMgr : }; - - } // namespace rocksdb #endif // ROCKSDB_LITE

1 0

[Commits] 70e12615732: Make rocksdb.range_locking test to be run in forward and backward-ordered CFs
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 70e12615732d658939972b24da162de8d57086f0 (fb-prod201801-226-g70e12615732) parent(s): cd66b2e5f14a40f581b639f64ed74ad3b0f9f37d author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 13:17:06 +0300 message: Make rocksdb.range_locking test to be run in forward and backward-ordered CFs (one test that makes assumptions about physical record ordering is only enabled in forward ordered cfs) --- mysql-test/suite/rocksdb/r/range_locking.result | 44 ++- .../suite/rocksdb/r/range_locking_rev_cf.result | 341 +++++++++++++++++ mysql-test/suite/rocksdb/t/range_locking.inc | 414 +++++++++++++++++++++ mysql-test/suite/rocksdb/t/range_locking.test | 386 +------------------ .../suite/rocksdb/t/range_locking_rev_cf.test | 12 + rocksdb | 2 +- 6 files changed, 803 insertions(+), 396 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index 439e9c6fb00..0518260f53a 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -2,8 +2,9 @@ show variables like 'rocksdb_use_range_locking'; Variable_name Value rocksdb_use_range_locking ON create table t1 ( -pk int primary key, -a int +pk int, +a int, +primary key (pk) comment 'default' ) engine=rocksdb; insert into t1 values (10,10),(20,20),(30,30); @@ -54,9 +55,10 @@ connection con1; rollback; ## Test that locks are not released when a statement in the transaction create table t2 ( -pk int primary key, +pk int, a int, -unique key(a) +primary key (pk) comment 'default', +unique key(a) comment 'default' ) engine=rocksdb; insert into t2 values (1,1),(2,2); begin; @@ -79,8 +81,9 @@ drop table t1; # Test INFORMATION_SCHEMA.lock_info in range-locking mode # create table t1 ( -pk int primary key, -a int +pk int, +a int, +primary key (pk) comment 'default' ) engine=rocksdb; insert into t1 values (10,10),(20,20),(30,30); @@ -117,7 +120,7 @@ create table t1 ( kp1 int not null, kp2 int not null, a int, -primary key(kp1, kp2) +primary key(kp1, kp2) comment 'default' ) engine=rocksdb; insert into t1 select 1, a, 1234 from t0; insert into t1 select 2, a, 1234 from t0; @@ -157,7 +160,7 @@ create table t1 ( kp1 int not null, kp2 int not null, a int, -key(kp1, kp2) +key(kp1, kp2) comment 'default' ) engine=rocksdb; insert into t1 select 1, a, 1234 from t0; insert into t1 values (2, 3, 1234); @@ -229,7 +232,11 @@ drop table t1; # # Same test as above, but check the range scan # -create table t1 (pk int primary key, a int) engine=rocksdb; +create table t1 ( +pk int, +a int, +primary key (pk) comment 'default' +) engine=rocksdb; insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); connect con1,localhost,root,,; # TRX1: Start, Allocate a snapshot @@ -267,7 +274,11 @@ drop table t1; # # Same as above, but test SELECT FOR UPDATE. # -create table t1 (pk int primary key, a int) engine=rocksdb; +create table t1 ( +pk int, +a int, +primary key (pk) comment 'default' +) engine=rocksdb; insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); connect con1,localhost,root,,; # TRX1: Start, Allocate a snapshot @@ -305,7 +316,12 @@ drop table t1; # Another no-snapshot-checking test, this time for single-statement # transaction # -create table t1 (pk int primary key, a int, name varchar(16)) engine=rocksdb; +create table t1 ( +pk int, +a int, +name varchar(16), +primary key(pk) comment 'default' +) engine=rocksdb; insert into t1 values (1,1, 'row1'), (2,2,'row2'); connect con1,localhost,root,,; connection con1; @@ -339,7 +355,11 @@ drop table t1; # # Check that I_S.processlist.state is set correctly now. # -create table t1(pk int primary key, a int) engine=rocksdb; +create table t1( +pk int, +a int, +primary key(pk) comment 'default' +) engine=rocksdb; insert into t1 values (1,1),(2,2),(3,3); begin; select * from t1 where pk=2 for update; diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result new file mode 100644 index 00000000000..444c51162cd --- /dev/null +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -0,0 +1,341 @@ +show variables like 'rocksdb_use_range_locking'; +Variable_name Value +rocksdb_use_range_locking ON +create table t1 ( +pk int, +a int, +primary key (pk) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 values +(10,10),(20,20),(30,30); +connect con1,localhost,root,,; +connect con2,localhost,root,,; +### Test: check that range lock inhibits a point lock +connection con1; +begin; +select * from t1 where pk between 5 and 25 for update; +pk a +10 10 +20 20 +connection con2; +insert into t1 values (15,15); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY +connection con1; +rollback; +## Test: check that range lock inhibits another range lock +## (It should not, but currently it does) +connection con1; +begin; +select * from t1 where pk between 5 and 25 for update; +pk a +10 10 +20 20 +connection con2; +begin; +select * from t1 where pk between 15 and 35 for update; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY +rollback; +connection con1; +rollback; +## Test: check that regular read does not get a range lock +connection con1; +begin; +select * from t1 where pk between 5 and 25; +pk a +10 10 +20 20 +connection con2; +begin; +select * from t1 where pk between 15 and 35 for update; +pk a +20 20 +30 30 +rollback; +connection con1; +rollback; +## Test that locks are not released when a statement in the transaction +create table t2 ( +pk int, +a int, +primary key (pk) comment 'rev:cf1', +unique key(a) comment '' +) engine=rocksdb; +insert into t2 values (1,1),(2,2); +begin; +insert into t2 values (3,3); +insert into t2 values (10,2); +ERROR 23000: Duplicate entry '2' for key 'a' +connection con2; +begin; +select * from t2 where pk=3 for update; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t2.PRIMARY +rollback; +connection con1; +rollback; +drop table t2; +connection default; +disconnect con1; +disconnect con2; +drop table t1; +# +# Test INFORMATION_SCHEMA.lock_info in range-locking mode +# +create table t1 ( +pk int, +a int, +primary key (pk) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 values +(10,10),(20,20),(30,30); +begin; +select * from t1 where pk=10 for update; +pk a +10 10 +select * from information_schema.rocksdb_locks; +COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE +0 15 00000001078000000a X +delete from t1 where pk between 25 and 40; +select * from information_schema.rocksdb_locks; +COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE +0 15 000000010780000019 - 010000010780000028 X +0 15 00000001078000000a X +rollback; +begin; +# The following will show a range lock on 2-9 and also a point lock on 10. +# This is how things currently work. +select * from t1 where pk between 2 and 9 for update; +pk a +select * from information_schema.rocksdb_locks; +COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE +0 16 00000001078000000a X +0 16 000000010780000002 - 010000010780000009 X +rollback; +drop table t1; +# +# MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( +kp1 int not null, +kp2 int not null, +a int, +primary key(kp1, kp2) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 select 1, a, 1234 from t0; +insert into t1 select 2, a, 1234 from t0; +insert into t1 select 3, a, 1234 from t0; +connect con1,localhost,root,,; +connection con1; +begin; +select * from t1 where kp1=2 for update; +kp1 kp2 a +2 0 1234 +2 1 1234 +2 2 1234 +2 3 1234 +2 4 1234 +2 5 1234 +2 6 1234 +2 7 1234 +2 8 1234 +2 9 1234 +connection default; +# The lock on kp1=2 should inhibit the following INSERT: +insert into t1 values ( 2,5,9999); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY +rollback; +connection con1; +rollback; +connection default; +disconnect con1; +drop table t0,t1; +# +# Test that locks on ranges on non-unique secondary keys inhibit +# modifications of the contents of these ranges +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( +kp1 int not null, +kp2 int not null, +a int, +key(kp1, kp2) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 select 1, a, 1234 from t0; +insert into t1 values (2, 3, 1234); +insert into t1 values (2, 5, 1234); +insert into t1 values (2, 7, 1234); +insert into t1 select 3, a, 1234 from t0; +connect con1,localhost,root,,; +connection con1; +begin; +explain +select * from t1 where kp1=2 for update; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref kp1 kp1 4 const # NULL +select * from t1 where kp1=2 for update; +kp1 kp2 a +2 3 1234 +2 5 1234 +2 7 1234 +connection default; +begin; +insert into t1 values (2, 9, 9999); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.kp1 +delete from t1 where kp1=2 and kp2=5; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.kp1 +update t1 set kp1=333 where kp1=2 and kp2=3; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.kp1 +update t1 set kp1=2 where kp1=1 and kp2=8; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.kp1 +rollback; +connection con1; +rollback; +disconnect con1; +connection default; +drop table t0,t1; +# +# Transaction isolation test +# +create table t1 (pk int primary key, a int) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +connect con1,localhost,root,,; +# TRX1: Start, Allocate a snapshot +connection con1; +begin; +select * from t1; +pk a +1 1 +2 2 +3 3 +# TRX2: Make a change that TRX1 will not see +connection default; +update t1 set a=2222 where pk=2; +# TRX1: Now, make a change that would overwrite TRX2'x change and commit +connection con1; +update t1 set a=a+1 where pk=2; +commit; +# Examine the result: +# pk=2, a=2223 means UPDATE in TRX1 used "read committed" (InnoDB-like isolation) +# pk=2, a=3 means UPDATE in TRX1 silently overwrote TRX2 +# (and with key tracking, one would get an error on the second UPDATE) +connection default; +select * from t1; +pk a +1 1 +2 2223 +3 3 +disconnect con1; +connection default; +drop table t1; +# +# Same test as above, but check the range scan +# +create table t1 ( +pk int, +a int, +primary key (pk) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); +connect con1,localhost,root,,; +# TRX1: Start, Allocate a snapshot +connection con1; +begin; +select * from t1; +pk a +6 6 +5 5 +4 4 +3 3 +2 2 +1 1 +# TRX2: Make a change that TRX1 will not see +connection default; +update t1 set a=2222 where pk between 3 and 5; +# TRX1: Now, make a change that would overwrite TRX2'x change and commit +connection con1; +update t1 set a=a+1 where pk between 3 and 5; +commit; +# Examine the result: +# pk={3,4,5} a=2223 means UPDATE in TRX1 used "read committed" (InnoDB-like isolation) +connection default; +select * from t1; +pk a +6 6 +5 2223 +4 2223 +3 2223 +2 2 +1 1 +disconnect con1; +connection default; +drop table t1; +# +# Same as above, but test SELECT FOR UPDATE. +# +create table t1 ( +pk int, +a int, +primary key (pk) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); +connect con1,localhost,root,,; +# TRX1: Start, Allocate a snapshot +connection con1; +begin; +select * from t1; +pk a +6 6 +5 5 +4 4 +3 3 +2 2 +1 1 +# TRX2: Make a change that TRX1 will not see +connection default; +update t1 set a=222 where pk=2; +update t1 set a=333 where pk=3; +# TRX1: Check what select [FOR UPDATE] sees +connection con1; +select * from t1 where pk in (2,3); +pk a +2 2 +3 3 +select * from t1 where pk=2 for update; +pk a +2 222 +select * from t1 where pk=2; +pk a +2 2 +commit; +disconnect con1; +connection default; +drop table t1; +# +# Check that I_S.processlist.state is set correctly now. +# +create table t1( +pk int, +a int, +primary key(pk) comment 'rev:cf1' +) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +begin; +select * from t1 where pk=2 for update; +pk a +2 2 +connect con1,localhost,root,,; +begin; +set rocksdb_lock_wait_timeout=300; +select * from t1 where pk=2 for update; +connection default; +# Now, will wait until we see con1 have state="Waiting for row lock" +rollback; +connection con1; +pk a +2 2 +rollback; +disconnect con1; +connection default; +drop table t1; diff --git a/mysql-test/suite/rocksdb/t/range_locking.inc b/mysql-test/suite/rocksdb/t/range_locking.inc new file mode 100644 index 00000000000..9c826811695 --- /dev/null +++ b/mysql-test/suite/rocksdb/t/range_locking.inc @@ -0,0 +1,414 @@ +# +# Range locking tests. +# + +--source include/have_rocksdb.inc +--source suite/rocksdb/include/have_range_locking.inc + +--enable_connect_log + + +show variables like 'rocksdb_use_range_locking'; + +# Test 1: see that ranges are indeed lockied. + +eval create table t1 ( + pk int, + a int, + primary key (pk) comment '$pk_cf' +) engine=rocksdb; + +insert into t1 values +(10,10),(20,20),(30,30); + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); + +--echo ### Test: check that range lock inhibits a point lock +connection con1; +begin; +select * from t1 where pk between 5 and 25 for update; + +connection con2; +--error ER_LOCK_WAIT_TIMEOUT +insert into t1 values (15,15); + +connection con1; +rollback; + +--echo ## Test: check that range lock inhibits another range lock +--echo ## (It should not, but currently it does) +connection con1; +begin; +select * from t1 where pk between 5 and 25 for update; + +connection con2; +begin; +--error ER_LOCK_WAIT_TIMEOUT +select * from t1 where pk between 15 and 35 for update; +rollback; + +connection con1; +rollback; + +--echo ## Test: check that regular read does not get a range lock +connection con1; +begin; +select * from t1 where pk between 5 and 25; + +connection con2; +begin; +# This must not block +select * from t1 where pk between 15 and 35 for update; +rollback; + +connection con1; +rollback; + +--echo ## Test that locks are not released when a statement in the transaction +eval +create table t2 ( + pk int, + a int, + primary key (pk) comment '$pk_cf', + unique key(a) comment '$sk_cf' +) engine=rocksdb; + +insert into t2 values (1,1),(2,2); + +begin; +insert into t2 values (3,3); +--error ER_DUP_ENTRY +insert into t2 values (10,2); + +connection con2; +begin; +# This must time out: +--error ER_LOCK_WAIT_TIMEOUT +select * from t2 where pk=3 for update; + +rollback; +connection con1; +rollback; +drop table t2; + +# Cleanup +connection default; +disconnect con1; +disconnect con2; +drop table t1; + +--echo # +--echo # Test INFORMATION_SCHEMA.lock_info in range-locking mode +--echo # +eval +create table t1 ( + pk int, + a int, + primary key (pk) comment '$pk_cf' +) engine=rocksdb; + +insert into t1 values +(10,10),(20,20),(30,30); + +begin; +select * from t1 where pk=10 for update; + +select * from information_schema.rocksdb_locks; + +delete from t1 where pk between 25 and 40; +select * from information_schema.rocksdb_locks; + +rollback; +begin; +--echo # The following will show a range lock on 2-9 and also a point lock on 10. +--echo # This is how things currently work. +select * from t1 where pk between 2 and 9 for update; +select * from information_schema.rocksdb_locks; +rollback; + +drop table t1; + +--echo # +--echo # MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +eval +create table t1 ( + kp1 int not null, + kp2 int not null, + a int, + primary key(kp1, kp2) comment '$pk_cf' +) engine=rocksdb; + +insert into t1 select 1, a, 1234 from t0; +insert into t1 select 2, a, 1234 from t0; +insert into t1 select 3, a, 1234 from t0; + +connect (con1,localhost,root,,); +connection con1; + +begin; +select * from t1 where kp1=2 for update; + +connection default; +--echo # The lock on kp1=2 should inhibit the following INSERT: +--error ER_LOCK_WAIT_TIMEOUT +insert into t1 values ( 2,5,9999); +rollback; + +connection con1; +rollback; +connection default; +disconnect con1; +drop table t0,t1; + +--echo # +--echo # Test that locks on ranges on non-unique secondary keys inhibit +--echo # modifications of the contents of these ranges +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +eval +create table t1 ( + kp1 int not null, + kp2 int not null, + a int, + key(kp1, kp2) comment '$pk_cf' +) engine=rocksdb; + +insert into t1 select 1, a, 1234 from t0; +insert into t1 values (2, 3, 1234); +insert into t1 values (2, 5, 1234); +insert into t1 values (2, 7, 1234); +insert into t1 select 3, a, 1234 from t0; + +connect (con1,localhost,root,,); +connection con1; +begin; +--replace_column 9 # +explain +select * from t1 where kp1=2 for update; +select * from t1 where kp1=2 for update; + +connection default; +begin; +--error ER_LOCK_WAIT_TIMEOUT +insert into t1 values (2, 9, 9999); + +--error ER_LOCK_WAIT_TIMEOUT +delete from t1 where kp1=2 and kp2=5; + +# Update that "moves a row away" from the locked range +--error ER_LOCK_WAIT_TIMEOUT +update t1 set kp1=333 where kp1=2 and kp2=3; + +# Update that "moves a row into" the locked range +--error ER_LOCK_WAIT_TIMEOUT +update t1 set kp1=2 where kp1=1 and kp2=8; + +rollback; + +connection con1; +rollback; +disconnect con1; +connection default; +drop table t0,t1; + +--echo # +--echo # Transaction isolation test +--echo # + +create table t1 (pk int primary key, a int) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3); + +connect (con1,localhost,root,,); + +--echo # TRX1: Start, Allocate a snapshot +connection con1; +begin; +select * from t1; + +--echo # TRX2: Make a change that TRX1 will not see +connection default; +update t1 set a=2222 where pk=2; + +--echo # TRX1: Now, make a change that would overwrite TRX2'x change and commit +connection con1; +update t1 set a=a+1 where pk=2; +commit; + +--echo # Examine the result: +--echo # pk=2, a=2223 means UPDATE in TRX1 used "read committed" (InnoDB-like isolation) +--echo # pk=2, a=3 means UPDATE in TRX1 silently overwrote TRX2 +--echo # (and with key tracking, one would get an error on the second UPDATE) +connection default; +select * from t1; + +disconnect con1; +connection default; +drop table t1; + +--echo # +--echo # Same test as above, but check the range scan +--echo # + +eval +create table t1 ( + pk int, + a int, + primary key (pk) comment '$pk_cf' +) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); + +connect (con1,localhost,root,,); + +--echo # TRX1: Start, Allocate a snapshot +connection con1; +begin; +select * from t1; + +--echo # TRX2: Make a change that TRX1 will not see +connection default; +update t1 set a=2222 where pk between 3 and 5; + +--echo # TRX1: Now, make a change that would overwrite TRX2'x change and commit +connection con1; +update t1 set a=a+1 where pk between 3 and 5; +commit; + +--echo # Examine the result: +--echo # pk={3,4,5} a=2223 means UPDATE in TRX1 used "read committed" (InnoDB-like isolation) +connection default; +select * from t1; + +disconnect con1; +connection default; +drop table t1; + +--echo # +--echo # Same as above, but test SELECT FOR UPDATE. +--echo # +eval +create table t1 ( + pk int, + a int, + primary key (pk) comment '$pk_cf' +) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); + +connect (con1,localhost,root,,); + +--echo # TRX1: Start, Allocate a snapshot +connection con1; +begin; +select * from t1; + +--echo # TRX2: Make a change that TRX1 will not see +connection default; +update t1 set a=222 where pk=2; +update t1 set a=333 where pk=3; + +--echo # TRX1: Check what select [FOR UPDATE] sees +connection con1; +select * from t1 where pk in (2,3); +select * from t1 where pk=2 for update; +select * from t1 where pk=2; + +commit; + +disconnect con1; +connection default; +drop table t1; + +if (!$PK_USES_REVERSE_CF) { +--echo # +--echo # Another no-snapshot-checking test, this time for single-statement +--echo # transaction +--echo # +eval +create table t1 ( + pk int, + a int, + name varchar(16), + primary key(pk) comment '$pk_cf' +) engine=rocksdb; +insert into t1 values (1,1, 'row1'), (2,2,'row2'); + +connect (con1,localhost,root,,); +connection con1; +select get_lock('row1', 100); + +connection default; + +--echo # The following will read the first row (1,1,'row1'), and stop. + +send update t1 set a=a+100 where get_lock(name, 1000)=1; + +# Wait till the default connection has stopped: +connection con1; + +let $wait_condition= + SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = "User lock" + AND INFO = "update t1 set a=a+100 where get_lock(name, 1000)=1"; +--source include/wait_condition.inc + +# Update the second row +update t1 set a=5 where pk=2; + +select release_lock('row1'); + +connection default; +reap; + +--echo # Look at the row with pk=2: +--echo # 2, 105, row2 - means the UPDATE was reading current data (Correct) +--echo # 2, 102, row - means the UPDATE read the snapshot (incorrect) +select * from t1; + +--echo # Try releasing both locks (in 5.6, we will be holding only the second one) +select release_lock(name) from t1; + +disconnect con1; +connection default; +drop table t1; +} + +--echo # +--echo # Check that I_S.processlist.state is set correctly now. +--echo # +eval +create table t1( + pk int, + a int, + primary key(pk) comment '$pk_cf' +) engine=rocksdb; +insert into t1 values (1,1),(2,2),(3,3); + +begin; +select * from t1 where pk=2 for update; + +--connect (con1,localhost,root,,) +begin; +set rocksdb_lock_wait_timeout=300; +send select * from t1 where pk=2 for update; + +connection default; +--echo # Now, will wait until we see con1 have state="Waiting for row lock" +let $wait_condition= + SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = "Waiting for row lock" + AND INFO = "select * from t1 where pk=2 for update"; +--source include/wait_condition.inc + +rollback; +connection con1; +--reap +rollback; + +disconnect con1; +connection default; +drop table t1; + diff --git a/mysql-test/suite/rocksdb/t/range_locking.test b/mysql-test/suite/rocksdb/t/range_locking.test index df926a2920b..694b13215af 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.test +++ b/mysql-test/suite/rocksdb/t/range_locking.test @@ -1,386 +1,6 @@ -# -# Range locking tests. -# ---source include/have_rocksdb.inc ---source suite/rocksdb/include/have_range_locking.inc ---enable_connect_log - - -show variables like 'rocksdb_use_range_locking'; - -# Test 1: see that ranges are indeed lockied. - -create table t1 ( - pk int primary key, - a int -) engine=rocksdb; - -insert into t1 values -(10,10),(20,20),(30,30); - -connect (con1,localhost,root,,); -connect (con2,localhost,root,,); - ---echo ### Test: check that range lock inhibits a point lock -connection con1; -begin; -select * from t1 where pk between 5 and 25 for update; - -connection con2; ---error ER_LOCK_WAIT_TIMEOUT -insert into t1 values (15,15); - -connection con1; -rollback; - ---echo ## Test: check that range lock inhibits another range lock ---echo ## (It should not, but currently it does) -connection con1; -begin; -select * from t1 where pk between 5 and 25 for update; - -connection con2; -begin; ---error ER_LOCK_WAIT_TIMEOUT -select * from t1 where pk between 15 and 35 for update; -rollback; - -connection con1; -rollback; - ---echo ## Test: check that regular read does not get a range lock -connection con1; -begin; -select * from t1 where pk between 5 and 25; - -connection con2; -begin; -# This must not block -select * from t1 where pk between 15 and 35 for update; -rollback; - -connection con1; -rollback; - ---echo ## Test that locks are not released when a statement in the transaction -create table t2 ( - pk int primary key, - a int, - unique key(a) -) engine=rocksdb; - -insert into t2 values (1,1),(2,2); - -begin; -insert into t2 values (3,3); ---error ER_DUP_ENTRY -insert into t2 values (10,2); - -connection con2; -begin; -# This must time out: ---error ER_LOCK_WAIT_TIMEOUT -select * from t2 where pk=3 for update; - -rollback; -connection con1; -rollback; -drop table t2; - -# Cleanup -connection default; -disconnect con1; -disconnect con2; -drop table t1; - ---echo # ---echo # Test INFORMATION_SCHEMA.lock_info in range-locking mode ---echo # -create table t1 ( - pk int primary key, - a int -) engine=rocksdb; - -insert into t1 values -(10,10),(20,20),(30,30); - -begin; -select * from t1 where pk=10 for update; - -select * from information_schema.rocksdb_locks; - -delete from t1 where pk between 25 and 40; -select * from information_schema.rocksdb_locks; - -rollback; -begin; ---echo # The following will show a range lock on 2-9 and also a point lock on 10. ---echo # This is how things currently work. -select * from t1 where pk between 2 and 9 for update; -select * from information_schema.rocksdb_locks; -rollback; - -drop table t1; - ---echo # ---echo # MDEV-18104: MyRocks-Gap-Lock: range locking bounds are incorrect for multi-part keys ---echo # - -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -create table t1 ( - kp1 int not null, - kp2 int not null, - a int, - primary key(kp1, kp2) -) engine=rocksdb; - -insert into t1 select 1, a, 1234 from t0; -insert into t1 select 2, a, 1234 from t0; -insert into t1 select 3, a, 1234 from t0; - -connect (con1,localhost,root,,); -connection con1; - -begin; -select * from t1 where kp1=2 for update; - -connection default; ---echo # The lock on kp1=2 should inhibit the following INSERT: ---error ER_LOCK_WAIT_TIMEOUT -insert into t1 values ( 2,5,9999); -rollback; - -connection con1; -rollback; -connection default; -disconnect con1; -drop table t0,t1; - ---echo # ---echo # Test that locks on ranges on non-unique secondary keys inhibit ---echo # modifications of the contents of these ranges ---echo # - -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -create table t1 ( - kp1 int not null, - kp2 int not null, - a int, - key(kp1, kp2) -) engine=rocksdb; - -insert into t1 select 1, a, 1234 from t0; -insert into t1 values (2, 3, 1234); -insert into t1 values (2, 5, 1234); -insert into t1 values (2, 7, 1234); -insert into t1 select 3, a, 1234 from t0; - -connect (con1,localhost,root,,); -connection con1; -begin; ---replace_column 9 # -explain -select * from t1 where kp1=2 for update; -select * from t1 where kp1=2 for update; - -connection default; -begin; ---error ER_LOCK_WAIT_TIMEOUT -insert into t1 values (2, 9, 9999); - ---error ER_LOCK_WAIT_TIMEOUT -delete from t1 where kp1=2 and kp2=5; - -# Update that "moves a row away" from the locked range ---error ER_LOCK_WAIT_TIMEOUT -update t1 set kp1=333 where kp1=2 and kp2=3; - -# Update that "moves a row into" the locked range ---error ER_LOCK_WAIT_TIMEOUT -update t1 set kp1=2 where kp1=1 and kp2=8; - -rollback; - -connection con1; -rollback; -disconnect con1; -connection default; -drop table t0,t1; - ---echo # ---echo # Transaction isolation test ---echo # - -create table t1 (pk int primary key, a int) engine=rocksdb; -insert into t1 values (1,1),(2,2),(3,3); - -connect (con1,localhost,root,,); - ---echo # TRX1: Start, Allocate a snapshot -connection con1; -begin; -select * from t1; - ---echo # TRX2: Make a change that TRX1 will not see -connection default; -update t1 set a=2222 where pk=2; - ---echo # TRX1: Now, make a change that would overwrite TRX2'x change and commit -connection con1; -update t1 set a=a+1 where pk=2; -commit; - ---echo # Examine the result: ---echo # pk=2, a=2223 means UPDATE in TRX1 used "read committed" (InnoDB-like isolation) ---echo # pk=2, a=3 means UPDATE in TRX1 silently overwrote TRX2 ---echo # (and with key tracking, one would get an error on the second UPDATE) -connection default; -select * from t1; - -disconnect con1; -connection default; -drop table t1; - ---echo # ---echo # Same test as above, but check the range scan ---echo # - -create table t1 (pk int primary key, a int) engine=rocksdb; -insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); - -connect (con1,localhost,root,,); - ---echo # TRX1: Start, Allocate a snapshot -connection con1; -begin; -select * from t1; - ---echo # TRX2: Make a change that TRX1 will not see -connection default; -update t1 set a=2222 where pk between 3 and 5; - ---echo # TRX1: Now, make a change that would overwrite TRX2'x change and commit -connection con1; -update t1 set a=a+1 where pk between 3 and 5; -commit; - ---echo # Examine the result: ---echo # pk={3,4,5} a=2223 means UPDATE in TRX1 used "read committed" (InnoDB-like isolation) -connection default; -select * from t1; - -disconnect con1; -connection default; -drop table t1; - ---echo # ---echo # Same as above, but test SELECT FOR UPDATE. ---echo # -create table t1 (pk int primary key, a int) engine=rocksdb; -insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); - -connect (con1,localhost,root,,); - ---echo # TRX1: Start, Allocate a snapshot -connection con1; -begin; -select * from t1; - ---echo # TRX2: Make a change that TRX1 will not see -connection default; -update t1 set a=222 where pk=2; -update t1 set a=333 where pk=3; - ---echo # TRX1: Check what select [FOR UPDATE] sees -connection con1; -select * from t1 where pk in (2,3); -select * from t1 where pk=2 for update; -select * from t1 where pk=2; - -commit; - -disconnect con1; -connection default; -drop table t1; - ---echo # ---echo # Another no-snapshot-checking test, this time for single-statement ---echo # transaction ---echo # - -create table t1 (pk int primary key, a int, name varchar(16)) engine=rocksdb; -insert into t1 values (1,1, 'row1'), (2,2,'row2'); - -connect (con1,localhost,root,,); -connection con1; -select get_lock('row1', 100); - -connection default; - ---echo # The following will read the first row (1,1,'row1'), and stop. - -send update t1 set a=a+100 where get_lock(name, 1000)=1; - -# Wait till the default connection has stopped: -connection con1; - -let $wait_condition= - SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = "User lock" - AND INFO = "update t1 set a=a+100 where get_lock(name, 1000)=1"; ---source include/wait_condition.inc - -# Update the second row -update t1 set a=5 where pk=2; - -select release_lock('row1'); - -connection default; -reap; - ---echo # Look at the row with pk=2: ---echo # 2, 105, row2 - means the UPDATE was reading current data (Correct) ---echo # 2, 102, row - means the UPDATE read the snapshot (incorrect) -select * from t1; - ---echo # Try releasing both locks (in 5.6, we will be holding only the second one) -select release_lock(name) from t1; - -disconnect con1; -connection default; -drop table t1; - - ---echo # ---echo # Check that I_S.processlist.state is set correctly now. ---echo # -create table t1(pk int primary key, a int) engine=rocksdb; -insert into t1 values (1,1),(2,2),(3,3); - -begin; -select * from t1 where pk=2 for update; - ---connect (con1,localhost,root,,) -begin; -set rocksdb_lock_wait_timeout=300; -send select * from t1 where pk=2 for update; - -connection default; ---echo # Now, will wait until we see con1 have state="Waiting for row lock" -let $wait_condition= - SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = "Waiting for row lock" - AND INFO = "select * from t1 where pk=2 for update"; ---source include/wait_condition.inc - -rollback; -connection con1; ---reap -rollback; - -disconnect con1; -connection default; -drop table t1; +--let pk_cf=default +--let sk_cf=default +--source range_locking.inc diff --git a/mysql-test/suite/rocksdb/t/range_locking_rev_cf.test b/mysql-test/suite/rocksdb/t/range_locking_rev_cf.test new file mode 100644 index 00000000000..8b993764235 --- /dev/null +++ b/mysql-test/suite/rocksdb/t/range_locking_rev_cf.test @@ -0,0 +1,12 @@ +# +# Range locking tests. +# + +--source include/have_rocksdb.inc +--source suite/rocksdb/include/have_range_locking.inc + +--let pk_cf=rev:cf1 +--let PK_USES_REVERSE_CF=1 + +--source range_locking.inc + diff --git a/rocksdb b/rocksdb index 248acd10346..9af13f0e966 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 248acd10346428b078aed780183ffe7f9f3c6896 +Subproject commit 9af13f0e966171ef8688484b2da7fa576a1b4574

1 0

[Commits] 9af13f0e9: Temporary: range locking should still assume foward ordering
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 9af13f0e966171ef8688484b2da7fa576a1b4574 (v5.8-1037-g9af13f0e9) parent(s): 248acd10346428b078aed780183ffe7f9f3c6896 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 13:13:38 +0300 message: Temporary: range locking should still assume foward ordering --- utilities/transactions/transaction_lock_mgr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index 9b6fd9381..c98803acb 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -1204,7 +1204,7 @@ void RangeLockMgr::AddColumnFamily(const ColumnFamilyHandle *cfh) { if (!strcmp(cfh->GetComparator()->Name(), "RocksDB_SE_v3.10")) ltree_cmp = &fw_cmp_; else if (!strcmp(cfh->GetComparator()->Name(),"rev:RocksDB_SE_v3.10")) - ltree_cmp = &bw_cmp_; + ltree_cmp = &fw_cmp_; // temporary: use the same ordering else { assert(false); ltree_cmp= nullptr;

1 0

[Commits] a529188: MDEV-17456 Malicious SUPER user can possibly change audit log configuration without leaving traces.
by holyfoot＠askmonty.org 28 Apr '19

28 Apr '19

revision-id: a529188e05da8060c95eeb4c8caef05adbd6cc6a (mariadb-10.2.23-88-ga529188) parent(s): cd26cdcd974725031e30393ff165fb0dfb365c4d committer: Alexey Botchkov timestamp: 2019-04-29 01:25:17 +0400 message: MDEV-17456 Malicious SUPER user can possibly change audit log configuration without leaving traces. The 'SET server_audit_logging ' statements should be logged no matter what. --- mysql-test/suite/plugins/r/server_audit.result | 3 +++ mysql-test/suite/plugins/t/server_audit.test | 3 +++ plugin/server_audit/server_audit.c | 7 ++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index 0709444..b8d2986 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -212,6 +212,8 @@ select 2; 2 2 drop table t1; +set global server_audit_logging= off; +set global server_audit_logging= on; set global server_audit_events=''; set global server_audit_query_log_limit= 15; select (1), (2), (3), (4); @@ -378,6 +380,7 @@ TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD FOR u1=<secret>',ID TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'CREATE USER u3 IDENTIFIED BY *****',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop user u1, u2, u3',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'insert into t1 values (1), (2)',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'set global server_audit_logging= off',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'set global server_audit_events=\'\'',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'set global serv',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'select (1), (2)',0 diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index fa23dc5..f19c8f5 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -136,6 +136,9 @@ select * from t1; select 2; drop table t1; +set global server_audit_logging= off; +set global server_audit_logging= on; + set global server_audit_events=''; set global server_audit_query_log_limit= 15; diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 2f9cd99..f03564e 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -15,7 +15,7 @@ #define PLUGIN_VERSION 0x104 -#define PLUGIN_STR_VERSION "1.4.4" +#define PLUGIN_STR_VERSION "1.4.5" #define _my_thread_var loc_thread_var @@ -1623,7 +1623,7 @@ static int log_statement_ex(const struct connection_info *cn, } if (query && !(events & EVENT_QUERY_ALL) && - (events & EVENT_QUERY)) + (events & EVENT_QUERY && !cn->log_always)) { const char *orig_query= query; @@ -2556,9 +2556,10 @@ static void log_current_query(MYSQL_THD thd) if (!ci_needs_setup(cn) && cn->query_length && FILTER(EVENT_QUERY) && do_log_user(cn->user)) { + cn->log_always= 1; log_statement_ex(cn, cn->query_time, thd_get_thread_id(thd), cn->query, cn->query_length, 0, "QUERY"); - cn->log_always= 1; + cn->log_always= 0; } }

1 0

[Commits] cd66b2e5f14: Update to latest rocksdb (Use a different locktree for each column family)
by Sergei Petrunia 28 Apr '19

28 Apr '19

revision-id: cd66b2e5f14a40f581b639f64ed74ad3b0f9f37d (fb-prod201801-225-gcd66b2e5f14) parent(s): 9921dbd4542c4fe1e5d28c611f48816e2568fffd author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-28 23:41:13 +0300 message: Update to latest rocksdb (Use a different locktree for each column family) --- rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocksdb b/rocksdb index 97b782b47ae..248acd10346 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 97b782b47ae55675e2b0132d6332824343fe141e +Subproject commit 248acd10346428b078aed780183ffe7f9f3c6896

1 0

[Commits] 248acd103: Use a different locktree for each column family
by Sergei Petrunia 28 Apr '19

28 Apr '19

revision-id: 248acd10346428b078aed780183ffe7f9f3c6896 (v5.8-1036-g248acd103) parent(s): 97b782b47ae55675e2b0132d6332824343fe141e author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-28 23:39:10 +0300 message: Use a different locktree for each column family This is a preparation for range locking to properly support reverse-ordered column families (and not just assume an identical ordering across the whole DB). --- .../transactions/pessimistic_transaction_db.cc | 6 +- utilities/transactions/transaction_lock_mgr.cc | 184 +++++++++++++++++---- utilities/transactions/transaction_lock_mgr.h | 33 ++-- 3 files changed, 178 insertions(+), 45 deletions(-) diff --git a/utilities/transactions/pessimistic_transaction_db.cc b/utilities/transactions/pessimistic_transaction_db.cc index e50fb1dad..e3ce435af 100644 --- a/utilities/transactions/pessimistic_transaction_db.cc +++ b/utilities/transactions/pessimistic_transaction_db.cc @@ -356,7 +356,7 @@ Status TransactionDB::WrapStackableDB( // allocate a LockMap for it. void PessimisticTransactionDB::AddColumnFamily( const ColumnFamilyHandle* handle) { - lock_mgr_->AddColumnFamily(handle->GetID()); + lock_mgr_->AddColumnFamily(handle); } Status PessimisticTransactionDB::CreateColumnFamily( @@ -370,7 +370,7 @@ Status PessimisticTransactionDB::CreateColumnFamily( s = db_->CreateColumnFamily(options, column_family_name, handle); if (s.ok()) { - lock_mgr_->AddColumnFamily((*handle)->GetID()); + lock_mgr_->AddColumnFamily(*handle); UpdateCFComparatorMap(*handle); } @@ -385,7 +385,7 @@ Status PessimisticTransactionDB::DropColumnFamily( Status s = db_->DropColumnFamily(column_family); if (s.ok()) { - lock_mgr_->RemoveColumnFamily(column_family->GetID()); + lock_mgr_->RemoveColumnFamily(column_family); } return s; diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index edec58261..9b6fd9381 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -189,7 +189,8 @@ size_t LockMap::GetStripe(const std::string& key) const { return stripe; } -void TransactionLockMgr::AddColumnFamily(uint32_t column_family_id) { +void TransactionLockMgr::AddColumnFamily(const ColumnFamilyHandle *cfh) { + uint32_t column_family_id= cfh->GetID(); InstrumentedMutexLock l(&lock_map_mutex_); if (lock_maps_.find(column_family_id) == lock_maps_.end()) { @@ -202,7 +203,8 @@ void TransactionLockMgr::AddColumnFamily(uint32_t column_family_id) { } } -void TransactionLockMgr::RemoveColumnFamily(uint32_t column_family_id) { +void TransactionLockMgr::RemoveColumnFamily(const ColumnFamilyHandle *cfh) { + uint32_t column_family_id= cfh->GetID(); // Remove lock_map for this column family. Since the lock map is stored // as a shared ptr, concurrent transactions can still keep using it // until they release their references to it. @@ -780,23 +782,31 @@ RangeLockMgrHandle* NewRangeLockManager( class RangeLockList: public PessimisticTransaction::LockStorage { public: virtual ~RangeLockList() { - buffer_.destroy(); + for(auto it : buffers_) { + it.second->destroy(); + } } RangeLockList() : releasing_locks_(false) { - buffer_.create(); } - void append(const DBT *left_key, const DBT *right_key) { + void append(uint32_t cf_id, const DBT *left_key, const DBT *right_key) { MutexLock l(&mutex_); // there's only one thread that calls this function. // the same thread will do lock release. assert(!releasing_locks_); - buffer_.append(left_key, right_key); + auto it= buffers_.find(cf_id); + if (it == buffers_.end()) { + // create a new one + //it->second.create(); + it= buffers_.emplace(cf_id, std::shared_ptr<toku::range_buffer>(new toku::range_buffer())).first; + it->second->create(); + } + else + it->second->append(left_key, right_key); } - // Ranges that the transaction is holding locks on - toku::range_buffer buffer_; + std::unordered_map<uint32_t, std::shared_ptr<toku::range_buffer>> buffers_; // Synchronization. See RangeLockMgr::UnLockAll for details port::Mutex mutex_; @@ -836,6 +846,8 @@ Status RangeLockMgr::TryRangeLock(PessimisticTransaction* txn, // locktree::kill_waiter call. Do we need this anymore? TransactionID wait_txn_id = txn->GetID(); + auto lt= get_locktree_by_cfid(column_family_id); + request.set(lt, (TXNID)txn, &start_key_dbt, &end_key_dbt, toku::lock_request::WRITE, false /* not a big txn */, (void*)wait_txn_id); @@ -911,7 +923,7 @@ Status RangeLockMgr::TryRangeLock(PessimisticTransaction* txn, txn->owned_locks= std::unique_ptr<RangeLockList>(new RangeLockList); } RangeLockList* range_list= (RangeLockList*)txn->owned_locks.get(); - range_list->append(&start_key_dbt, &end_key_dbt); + range_list->append(column_family_id, &start_key_dbt, &end_key_dbt); return Status::OK(); } @@ -948,6 +960,7 @@ range_lock_mgr_release_lock_int(toku::locktree *lt, void RangeLockMgr::UnLock(PessimisticTransaction* txn, uint32_t column_family_id, const std::string& key, Env*) { + auto lt= get_locktree_by_cfid(column_family_id); range_lock_mgr_release_lock_int(lt, txn, column_family_id, key); toku::lock_request::retry_all_lock_requests(lt, nullptr /* lock_wait_needed_callback */); } @@ -959,14 +972,14 @@ void RangeLockMgr::UnLock(const PessimisticTransaction* txn, for (auto& key_map_iter : *key_map) { uint32_t column_family_id = key_map_iter.first; auto& keys = key_map_iter.second; + auto lt= get_locktree_by_cfid(column_family_id); for (auto& key_iter : keys) { const std::string& key = key_iter.first; range_lock_mgr_release_lock_int(lt, txn, column_family_id, key); } + toku::lock_request::retry_all_lock_requests(lt, nullptr /* lock_wait_needed_callback */); } - - toku::lock_request::retry_all_lock_requests(lt, nullptr /* lock_wait_needed_callback */); } void RangeLockMgr::UnLockAll(const PessimisticTransaction* txn, Env*) { @@ -1012,13 +1025,18 @@ void RangeLockMgr::UnLockAll(const PessimisticTransaction* txn, Env*) { // not holding any locks, the lock tree might be in the STO-mode with // another transaction, and our attempt to release an empty set of locks // will cause an assertion failure. - if (range_list->buffer_.get_num_ranges()) - lt->release_locks((TXNID)txn, &range_list->buffer_, true); - range_list->buffer_.destroy(); - range_list->buffer_.create(); - range_list->releasing_locks_= false; + for (auto it: range_list->buffers_) { + if (it.second->get_num_ranges()) { + toku::locktree *lt = get_locktree_by_cfid(it.first); + lt->release_locks((TXNID)txn, it.second.get(), true); - toku::lock_request::retry_all_lock_requests(lt, nullptr /* lock_wait_needed_callback */); + it.second->destroy(); + it.second->create(); + + toku::lock_request::retry_all_lock_requests(lt, nullptr); + } + } + range_list->releasing_locks_= false; } } @@ -1082,13 +1100,21 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, return res; } +int RangeLockMgr::compare_dbt_endpoints_rev(__toku_db* db, void *arg, + const DBT *a_key, + const DBT *b_key) { + return -compare_dbt_endpoints(db, arg, a_key, b_key); +} + RangeLockMgr::RangeLockMgr(std::shared_ptr<TransactionDBMutexFactory> mutex_factory) : - mutex_factory_(mutex_factory) { - ltm.create(on_create, on_destroy, on_escalate, NULL, mutex_factory_); - cmp_.create(compare_dbt_endpoints, (void*)this, NULL); - DICTIONARY_ID dict_id = { .dictid = 1 }; - lt= ltm.get_lt(dict_id, cmp_, /* on_create_extra*/nullptr); + mutex_factory_(mutex_factory), + ltree_lookup_cache_(new ThreadLocalPtr(nullptr)) { + + ltm_.create(on_create, on_destroy, on_escalate, NULL, mutex_factory_); + + fw_cmp_.create(compare_dbt_endpoints, (void*)this, NULL); + bw_cmp_.create(compare_dbt_endpoints, (void*)this, NULL); } @@ -1102,7 +1128,7 @@ RangeLockMgr::RangeLockMgr(std::shared_ptr<TransactionDBMutexFactory> mutex_fact @param void* Callback context */ -void RangeLockMgr::on_escalate(TXNID txnid, const locktree*, +void RangeLockMgr::on_escalate(TXNID txnid, const locktree* lt, const range_buffer &buffer, void *) { auto txn= (PessimisticTransaction*)txnid; @@ -1119,12 +1145,17 @@ void RangeLockMgr::on_escalate(TXNID txnid, const locktree*, } // TODO: are we tracking this mem: lt->get_manager()->note_mem_released(trx_locks->ranges.buffer->total_memory_size()); - trx_locks->buffer_.destroy(); - trx_locks->buffer_.create(); + + uint32_t cf_id = (uint32_t)lt->get_dict_id().dictid; + + auto it= trx_locks->buffers_.find(cf_id); + it->second->destroy(); + it->second->create(); + toku::range_buffer::iterator iter(&buffer); toku::range_buffer::iterator::record rec; while (iter.current(&rec)) { - trx_locks->buffer_.append(rec.get_left_key(), rec.get_right_key()); + it->second->append(rec.get_left_key(), rec.get_right_key()); iter.next(); } // TODO: same as above: lt->get_manager()->note_mem_used(ranges.buffer->total_memory_size()); @@ -1132,16 +1163,18 @@ void RangeLockMgr::on_escalate(TXNID txnid, const locktree*, RangeLockMgr::~RangeLockMgr() { - if (lt) { - ltm.release_lt(lt); + //TODO: at this point, synchronization is not needed, right? + for (auto it: ltree_map_) { + ltm_.release_lt(it.second); } - ltm.destroy(); - cmp_.destroy(); + ltm_.destroy(); + fw_cmp_.destroy(); + bw_cmp_.destroy(); } uint64_t RangeLockMgr::get_escalation_count() { LTM_STATUS_S ltm_status_test; - ltm.get_status(&ltm_status_test); + ltm_.get_status(&ltm_status_test); // Searching status variable by its string name is how Toku's unit tests // do it (why didn't they make LTM_ESCALATION_COUNT constant visible?) @@ -1160,6 +1193,87 @@ uint64_t RangeLockMgr::get_escalation_count() { return key_status->value.num; } +void RangeLockMgr::AddColumnFamily(const ColumnFamilyHandle *cfh) { + uint32_t column_family_id= cfh->GetID(); + + InstrumentedMutexLock l(&ltree_map_mutex_); + if (ltree_map_.find(column_family_id) == ltree_map_.end()) { + DICTIONARY_ID dict_id = { .dictid = column_family_id }; + toku::comparator *ltree_cmp; + // "RocksDB_SE_v3.10" // BytewiseComparator() ,ReverseBytewiseComparator() + if (!strcmp(cfh->GetComparator()->Name(), "RocksDB_SE_v3.10")) + ltree_cmp = &fw_cmp_; + else if (!strcmp(cfh->GetComparator()->Name(),"rev:RocksDB_SE_v3.10")) + ltree_cmp = &bw_cmp_; + else { + assert(false); + ltree_cmp= nullptr; + } + toku::locktree *ltree= ltm_.get_lt(dict_id, *ltree_cmp, + /* on_create_extra*/nullptr); + ltree_map_.emplace(column_family_id, ltree); + } else { + // column_family already exists in lock map + assert(false); + } +} + +void RangeLockMgr::RemoveColumnFamily(const ColumnFamilyHandle *cfh) { + uint32_t column_family_id= cfh->GetID(); + // Remove lock_map for this column family. Since the lock map is stored + // as a shared ptr, concurrent transactions can still keep using it + // until they release their references to it. + { + InstrumentedMutexLock l(&ltree_map_mutex_); + + auto lock_maps_iter = ltree_map_.find(column_family_id); + assert(lock_maps_iter != ltree_map_.end()); + + ltm_.release_lt(lock_maps_iter->second); + + ltree_map_.erase(lock_maps_iter); + } // lock_map_mutex_ + + //TODO: why do we delete first and clear the caches second? Shouldn't this be + // done in the reverse order? (if we do it in the reverse order, how will we + // prevent somebody from re-populating the cache?) + + // Clear all thread-local caches. We collect a vector of caches but we dont + // really need them. + autovector<void*> local_caches; + ltree_lookup_cache_->Scrape(&local_caches, nullptr); +} + +toku::locktree *RangeLockMgr::get_locktree_by_cfid(uint32_t column_family_id) { + + // First check thread-local cache + if (ltree_lookup_cache_->Get() == nullptr) { + ltree_lookup_cache_->Reset(new LockTreeMap()); + } + + auto ltree_map_cache = static_cast<LockTreeMap*>(ltree_lookup_cache_->Get()); + + auto it = ltree_map_cache->find(column_family_id); + if (it != ltree_map_cache->end()) { + // Found lock map for this column family. + return it->second; + } + + // Not found in local cache, grab mutex and check shared LockMaps + InstrumentedMutexLock l(&ltree_map_mutex_); + + it = ltree_map_.find(column_family_id); + if (it == ltree_map_.end()) { + return nullptr; + } else { + // Found lock map. Store in thread-local cache and return. + //std::shared_ptr<LockMap>& lock_map = lock_map_iter->second; + ltree_map_cache->insert({column_family_id, it->second}); + return it->second; + } + + return nullptr; +} struct LOCK_PRINT_CONTEXT { BaseLockMgr::LockStatusData *data; @@ -1193,7 +1307,13 @@ void push_into_lock_status_data(void* param, const DBT *left, BaseLockMgr::LockStatusData RangeLockMgr::GetLockStatusData() { LockStatusData data; LOCK_PRINT_CONTEXT ctx = {&data, GetColumnFamilyID(my_txn_db_->DefaultColumnFamily()) }; - lt->dump_locks((void*)&ctx, push_into_lock_status_data); + + { + InstrumentedMutexLock l(&ltree_map_mutex_); + for (auto it : ltree_map_) { + it.second->dump_locks((void*)&ctx, push_into_lock_status_data); + } + } return data; } diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index 02c68964e..e41a4961a 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -60,8 +60,8 @@ class PessimisticTransactionDB; // class BaseLockMgr { public: - virtual void AddColumnFamily(uint32_t column_family_id) = 0; - virtual void RemoveColumnFamily(uint32_t column_family_id) = 0; + virtual void AddColumnFamily(const ColumnFamilyHandle *cfh) = 0; + virtual void RemoveColumnFamily(const ColumnFamilyHandle *cfh) = 0; virtual Status TryLock(PessimisticTransaction* txn, uint32_t column_family_id, @@ -97,11 +97,11 @@ class TransactionLockMgr : public BaseLockMgr { // Creates a new LockMap for this column family. Caller should guarantee // that this column family does not already exist. - void AddColumnFamily(uint32_t column_family_id); + void AddColumnFamily(const ColumnFamilyHandle *cfh); // Deletes the LockMap for this column family. Caller should guarantee that // this column family is no longer in use. - void RemoveColumnFamily(uint32_t column_family_id); + void RemoveColumnFamily(const ColumnFamilyHandle *cfh); // Attempt to lock key. If OK status is returned, the caller is responsible // for calling UnLock() on this key. @@ -199,8 +199,8 @@ class RangeLockMgr : public BaseLockMgr, public RangeLockMgrHandle { public: - void AddColumnFamily(uint32_t) override { /* do nothing */ } - void RemoveColumnFamily(uint32_t) override { /* do nothing */ } + void AddColumnFamily(const ColumnFamilyHandle *cfh) override; + void RemoveColumnFamily(const ColumnFamilyHandle *cfh) override; Status TryLock(PessimisticTransaction* txn, uint32_t column_family_id, const std::string& key, Env* env, bool exclusive) override ; @@ -240,7 +240,7 @@ class RangeLockMgr : int set_max_lock_memory(size_t max_lock_memory) override { - return ltm.set_max_lock_memory(max_lock_memory); + return ltm_.set_max_lock_memory(max_lock_memory); } uint64_t get_escalation_count() override; @@ -248,15 +248,28 @@ class RangeLockMgr : LockStatusData GetLockStatusData() override; private: - toku::locktree_manager ltm; - toku::locktree *lt; // only one tree for now + toku::locktree_manager ltm_; - toku::comparator cmp_; + toku::comparator fw_cmp_; + toku::comparator bw_cmp_; TransactionDB* my_txn_db_; std::shared_ptr<TransactionDBMutexFactory> mutex_factory_; + // Map from cf_id to locktree*. Can only be accessed while holding the + // ltree_map_mutex_. + using LockTreeMap = std::unordered_map<uint32_t, locktree*>; + LockTreeMap ltree_map_; + + InstrumentedMutex ltree_map_mutex_; + + // Per-thread cache of ltree_map_. + std::unique_ptr<ThreadLocalPtr> ltree_lookup_cache_; + + toku::locktree *get_locktree_by_cfid(uint32_t cf_id); + static int compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); + static int compare_dbt_endpoints_rev(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); // Callbacks static int on_create(locktree*, void*) { return 0; /* no error */ }

1 0

[Commits] cd26cdc: MDEV-19141 server_audit_excl_users accepts only values with less than 1024 chars.
by holyfoot＠askmonty.org 28 Apr '19

28 Apr '19

revision-id: cd26cdcd974725031e30393ff165fb0dfb365c4d (mariadb-10.2.23-87-gcd26cdc) parent(s): 00377147e3029b982cbc29d3f4477362c6e6fdb4 committer: Alexey Botchkov timestamp: 2019-04-29 00:11:48 +0400 message: MDEV-19141 server_audit_excl_users accepts only values with less than 1024 chars. Since this limit is imposed by the SHOW_VAR_FUNC_BUFF_SIZE, we just launch the error message. --- mysql-test/suite/plugins/r/server_audit.result | 14 ++++++ mysql-test/suite/plugins/t/server_audit.test | 8 ++++ plugin/server_audit/server_audit.c | 62 +++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index 0139276..0709444 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -21,6 +21,16 @@ set global server_audit_incl_users=null; set global server_audit_file_path='server_audit.log'; set global server_audit_output_type=file; set global server_audit_logging=on; +set global server_audit_incl_users= repeat("'root',", 10000); +ERROR 42000: Variable 'server_audit_incl_users' can't be set to the value of ''root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','roo' +show variables like 'server_audit_incl_users'; +Variable_name Value +server_audit_incl_users +set global server_audit_excl_users= repeat("'root',", 10000); +ERROR 42000: Variable 'server_audit_excl_users' can't be set to the value of ''root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','roo' +show variables like 'server_audit_excl_users'; +Variable_name Value +server_audit_excl_users connect con1,localhost,root,,mysql; connection default; disconnect con1; @@ -251,6 +261,10 @@ uninstall plugin server_audit; Warnings: Warning 1620 Plugin is busy and will be uninstalled on shutdown TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'set global server_audit_logging=on',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'set global server_audit_incl_users= repeat("\'root\',", 10000)',ID +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'show variables like \'server_audit_incl_users\'',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'set global server_audit_excl_users= repeat("\'root\',", 10000)',ID +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'show variables like \'server_audit_excl_users\'',0 TIME,HOSTNAME,root,localhost,ID,0,CONNECT,mysql,,0 TIME,HOSTNAME,root,localhost,ID,0,DISCONNECT,mysql,,0 TIME,HOSTNAME,no_such_user,localhost,ID,0,FAILED_CONNECT,,,ID diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index 4af1ed8..fa23dc5 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -13,6 +13,14 @@ set global server_audit_incl_users=null; set global server_audit_file_path='server_audit.log'; set global server_audit_output_type=file; set global server_audit_logging=on; + +--error ER_WRONG_VALUE_FOR_VAR +set global server_audit_incl_users= repeat("'root',", 10000); +show variables like 'server_audit_incl_users'; +--error ER_WRONG_VALUE_FOR_VAR +set global server_audit_excl_users= repeat("'root',", 10000); +show variables like 'server_audit_excl_users'; + --sleep 2 connect (con1,localhost,root,,mysql); connection default; diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 0995327..2f9cd99 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -335,6 +335,10 @@ static void update_file_rotations(MYSQL_THD thd, struct st_mysql_sys_var *var, void *var_ptr, const void *save); static void update_incl_users(MYSQL_THD thd, struct st_mysql_sys_var *var, void *var_ptr, const void *save); +static int check_incl_users(MYSQL_THD thd, struct st_mysql_sys_var *var, void *save, + struct st_mysql_value *value); +static int check_excl_users(MYSQL_THD thd, struct st_mysql_sys_var *var, void *save, + struct st_mysql_value *value); static void update_excl_users(MYSQL_THD thd, struct st_mysql_sys_var *var, void *var_ptr, const void *save); static void update_output_type(MYSQL_THD thd, struct st_mysql_sys_var *var, @@ -354,10 +358,10 @@ static void rotate_log(MYSQL_THD thd, struct st_mysql_sys_var *var, static MYSQL_SYSVAR_STR(incl_users, incl_users, PLUGIN_VAR_RQCMDARG, "Comma separated list of users to monitor.", - NULL, update_incl_users, NULL); + check_incl_users, update_incl_users, NULL); static MYSQL_SYSVAR_STR(excl_users, excl_users, PLUGIN_VAR_RQCMDARG, "Comma separated list of users to exclude from auditing.", - NULL, update_excl_users, NULL); + check_excl_users, update_excl_users, NULL); /* bits in the event filter. */ #define EVENT_CONNECT 1 #define EVENT_QUERY_ALL 2 @@ -2643,16 +2647,56 @@ static void update_file_rotate_size(MYSQL_THD thd __attribute__((unused)), } +static int check_users(void *save, struct st_mysql_value *value, + size_t s, const char *name) +{ + const char *users; + int len= 0; + + users= value->val_str(value, NULL, &len); + if ((size_t) len > s) + { + error_header(); + fprintf(stderr, + "server_audit_%s_users value can't be longer than %ld characters.\n", + name, s); + return 1; + } + *((const char**)save)= users; + return 0; +} + +static int check_incl_users(MYSQL_THD thd __attribute__((unused)), + struct st_mysql_sys_var *var __attribute__((unused)), + void *save, struct st_mysql_value *value) +{ + return check_users(save, value, sizeof(incl_user_buffer), "incl"); +} + +static int check_excl_users(MYSQL_THD thd __attribute__((unused)), + struct st_mysql_sys_var *var __attribute__((unused)), + void *save, struct st_mysql_value *value) +{ + return check_users(save, value, sizeof(excl_user_buffer), "excl"); +} + + static void update_incl_users(MYSQL_THD thd, struct st_mysql_sys_var *var __attribute__((unused)), void *var_ptr __attribute__((unused)), const void *save) { char *new_users= (*(char **) save) ? *(char **) save : empty_str; + size_t new_len= strlen(new_users) + 1; if (!maria_55_started || !debug_server_started) flogger_mutex_lock(&lock_operations); mark_always_logged(thd); - strncpy(incl_user_buffer, new_users, sizeof(incl_user_buffer)-1); - incl_user_buffer[sizeof(incl_user_buffer)-1]= 0; + + if (new_len > sizeof(incl_user_buffer)) + new_len= sizeof(incl_user_buffer); + + memcpy(incl_user_buffer, new_users, new_len - 1); + incl_user_buffer[new_len - 1]= 0; + incl_users= incl_user_buffer; user_coll_fill(&incl_user_coll, incl_users, &excl_user_coll, 1); error_header(); @@ -2667,11 +2711,17 @@ static void update_excl_users(MYSQL_THD thd __attribute__((unused)), void *var_ptr __attribute__((unused)), const void *save) { char *new_users= (*(char **) save) ? *(char **) save : empty_str; + size_t new_len= strlen(new_users) + 1; if (!maria_55_started || !debug_server_started) flogger_mutex_lock(&lock_operations); mark_always_logged(thd); - strncpy(excl_user_buffer, new_users, sizeof(excl_user_buffer)-1); - excl_user_buffer[sizeof(excl_user_buffer)-1]= 0; + + if (new_len > sizeof(excl_user_buffer)) + new_len= sizeof(excl_user_buffer); + + memcpy(excl_user_buffer, new_users, new_len - 1); + excl_user_buffer[new_len - 1]= 0; + excl_users= excl_user_buffer; user_coll_fill(&excl_user_coll, excl_users, &incl_user_coll, 0); error_header();

1 0

[Commits] d3534d5: MDEV-19351 statistics_for_command_is_needed: Conditional jump or move
by IgorBabaev 28 Apr '19

28 Apr '19

revision-id: d3534d5b45a419260d0292d72a6a21f2a2ba2582 (mariadb-10.1.38-144-gd3534d5) parent(s): d88dfd873232dafd480c2d0382b4ff3add9c936d author: Igor Babaev committer: Igor Babaev timestamp: 2019-04-27 21:31:04 -0700 message: MDEV-19351 statistics_for_command_is_needed: Conditional jump or move depends on uninitialised value Initialized THD::force_read_stats introduced in the patch for MDEV-17605. Leaving this field uninitialized in the constructor of the THD class may trigger reading statistical data that is not needed. --- sql/sql_class.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 639c7c1..6fa8c10 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -983,6 +983,7 @@ THD::THD(bool is_wsrep_applier) memset(&invoker_host, 0, sizeof(invoker_host)); prepare_derived_at_open= FALSE; create_tmp_table_for_derived= FALSE; + force_read_stats= FALSE; save_prep_leaf_list= FALSE; /* Restore THR_THD */ set_current_thd(old_THR_THD);

1 0