- commits - lists.mariadb.org

[Commits] 1b423e395: Range Locking: clean up comments
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 1b423e3954d932c4624337308e3e1cd98481a495 (v5.8-1041-g1b423e395) parent(s): bcc7923688446c58a21f4f0ab287b147f406abc2 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 23:29:02 +0300 message: Range Locking: clean up comments --- include/rocksdb/utilities/transaction.h | 62 ++++++++++++----------- include/rocksdb/utilities/transaction_db.h | 13 +++-- utilities/transactions/pessimistic_transaction.cc | 1 - utilities/transactions/transaction_lock_mgr.cc | 21 +++----- utilities/transactions/transaction_lock_mgr.h | 14 +++-- 5 files changed, 52 insertions(+), 59 deletions(-) diff --git a/include/rocksdb/utilities/transaction.h b/include/rocksdb/utilities/transaction.h index b0f1308ed..6b84d4a44 100644 --- a/include/rocksdb/utilities/transaction.h +++ b/include/rocksdb/utilities/transaction.h @@ -31,50 +31,52 @@ using TransactionID = uint64_t; Prefix ranges are introduced below. == Basic Ranges == - - Basic ranges can be defined over rowkeys. Key Comparator defines ordering of - rowkeys, a finite closed range is the same as range over numbers: + Let's start from basic ranges. Key Comparator defines ordering of rowkeys. + Then, one can specify finite closed ranges by just providing rowkeys of their + endpoints: lower_endpoint <= X <= upper_endpoint - The endpoints here are possible rowkey values. - - == Lexicographic-like ordering == - - A lexicographic-like ordering satisfies these criteria: - - 1.The ordering is prefix-based. If there are two keys in form + However our goal is to provide a richer set of endpoints. Read on. - key_a = {prefix_a suffix_a} - key_b = {prefix_b suffix_b} + == Lexicographic ordering == + A lexicographic (or dictionary) ordering satisfies these criteria: If there + are two keys in form + key_a = {prefix_a, suffix_a} + key_b = {prefix_b, suffix_b} and prefix_a < prefix_b then key_a < key_b. - 2. An empty string is less than any other value. From this it follows that - for any prefix and suffix, {prefix, suffix} > {prefix}. + == Prefix ranges == + With lexicographic ordering, one may want to define ranges in form - 3. The row comparison function is able to compare key prefixes. If the data - domain includes keys A and B, then the comparison function is able to compare - equal-length prefixes: + "prefix is $PREFIX" - min_len= min(byte_length(A), byte_length(B)); - cmp(Slice(A, min_len), Slice(B, min_len)) // this call is valid + which translates to a range in form - == Prefix ranges == + {$PREFIX, -infinity} < X < {$PREFIX, +infinity} + + where -infinity will compare less than any possible suffix, and +infinity + will compare as greater than any possible suffix. - With lexicographic-like ordering, one may wish to construct ranges from a - restriction in form prefix=P: - - the left endpoint would would be {P, inf_suffix=false} - - the right endpoint would be {P, inf_suffix=true}. + class Endpoint allows to define these kind of rangtes. - == Supported comparators == - BytewiseComparator and ReverseBytweiseComparator meet the lexicographic-like - ordering requirements. + == Notes == + BytewiseComparator and ReverseBytewiseComparator produce lexicographic + ordering. + + The row comparison function is able to compare key prefixes. If the data + domain includes keys A and B, then the comparison function is able to compare + equal-length prefixes: + + min_len= min(byte_length(A), byte_length(B)); + cmp(Slice(A, min_len), Slice(B, min_len)); // this call is valid - TODO: RangeLocking will refuse to work if any other comparator is used, - although other comparators meeting this property could be used as well. + == Other options == + As far as MyRocks is concerned, the alternative to prefix ranges would be to + support both open (non-inclusive) and closed (inclusive) range endpoints. */ class Endpoint { @@ -82,7 +84,7 @@ class Endpoint { Slice slice; /* - true : the key has an "infinity" suffix. A suffix that would compare as + true : the key has a "+infinity" suffix. A suffix that would compare as greater than any other suffix false : otherwise */ diff --git a/include/rocksdb/utilities/transaction_db.h b/include/rocksdb/utilities/transaction_db.h index 8f0e018f2..7ebac3e06 100644 --- a/include/rocksdb/utilities/transaction_db.h +++ b/include/rocksdb/utilities/transaction_db.h @@ -33,7 +33,8 @@ enum TxnDBWritePolicy { const uint32_t kInitialMaxDeadlocks = 5; -// A handle to control RangeLockMgr +// A handle to control RangeLockMgr (Range-based lock manager) from outside +// RocksDB class RangeLockMgrHandle { public: virtual int set_max_lock_memory(size_t max_lock_memory) = 0; @@ -41,12 +42,15 @@ class RangeLockMgrHandle { virtual ~RangeLockMgrHandle() {}; }; -// A factory function to create a Range Lock Manager +// A factory function to create a Range Lock Manager. The created object should +// be: +// 1. Passed in TransactionDBOptions::range_lock_mgr to open the database in +// range-locking mode +// 2. Used to control the lock manager when the DB is already open. RangeLockMgrHandle* NewRangeLockManager( std::shared_ptr<TransactionDBMutexFactory> mutex_factory ); - struct TransactionDBOptions { // Specifies the maximum number of keys that can be locked at the same time // per column family. @@ -210,7 +214,6 @@ struct DeadlockPath { bool empty() { return path.empty() && !limit_exceeded; } }; - class TransactionDB : public StackableDB { public: // Optimized version of ::Write that receives more optimization request such @@ -284,7 +287,7 @@ class TransactionDB : public StackableDB { GetLockStatusData() = 0; virtual std::vector<DeadlockPath> GetDeadlockInfoBuffer() = 0; virtual void SetDeadlockInfoBufferSize(uint32_t target_size) = 0; - + protected: // To Create an TransactionDB, call Open() // The ownership of db is transferred to the base StackableDB diff --git a/utilities/transactions/pessimistic_transaction.cc b/utilities/transactions/pessimistic_transaction.cc index ff653fa0e..082934e75 100644 --- a/utilities/transactions/pessimistic_transaction.cc +++ b/utilities/transactions/pessimistic_transaction.cc @@ -133,7 +133,6 @@ WriteCommittedTxn::WriteCommittedTxn(TransactionDB* txn_db, Status PessimisticTransaction::CommitBatch(WriteBatch* batch) { TransactionKeyMap keys_to_unlock; - Status s = LockBatch(batch, &keys_to_unlock); if (!s.ok()) { diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index f837f6a5a..b6c91393d 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -1180,18 +1180,6 @@ void RangeLockMgr::AddColumnFamily(const ColumnFamilyHandle *cfh) { InstrumentedMutexLock l(&ltree_map_mutex_); if (ltree_map_.find(column_family_id) == ltree_map_.end()) { DICTIONARY_ID dict_id = { .dictid = column_family_id }; - // "RocksDB_SE_v3.10" // BytewiseComparator() ,ReverseBytewiseComparator() - /* - toku::comparator *ltree_cmp; - if (!strcmp(cfh->GetComparator()->Name(), "RocksDB_SE_v3.10")) - ltree_cmp = &fw_cmp_; - else if (!strcmp(cfh->GetComparator()->Name(),"rev:RocksDB_SE_v3.10")) - ltree_cmp = &bw_cmp_; - else { - assert(false); - ltree_cmp= nullptr; - } - */ toku::comparator cmp; cmp.create(compare_dbt_endpoints, (void*)cfh->GetComparator(), NULL); toku::locktree *ltree= ltm_.get_lt(dict_id, cmp, @@ -1210,6 +1198,10 @@ void RangeLockMgr::RemoveColumnFamily(const ColumnFamilyHandle *cfh) { // Remove lock_map for this column family. Since the lock map is stored // as a shared ptr, concurrent transactions can still keep using it // until they release their references to it. + + // TODO^ if one can drop a column family while a transaction is holding a + // lock in it, is column family's comparator guaranteed to survive until + // all locks are released? (we depend on this). { InstrumentedMutexLock l(&ltree_map_mutex_); @@ -1263,9 +1255,8 @@ toku::locktree *RangeLockMgr::get_locktree_by_cfid(uint32_t column_family_id) { } struct LOCK_PRINT_CONTEXT { - BaseLockMgr::LockStatusData *data; - // this will not be needed when locks are per-column-family: - uint32_t cfh_id; + BaseLockMgr::LockStatusData *data; // Save locks here + uint32_t cfh_id; // Column Family whose tree we are traversing }; static diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index 1f5dbc634..608d6f34f 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -212,8 +212,8 @@ class RangeLockMgr : }; // Get a lock on a range - // (TODO: this allows to acquire exclusive range locks although they are not - // used ATM) + // @note only exclusive locks are currently supported (requesting a + // non-exclusive lock will get an exclusive one) Status TryRangeLock(PessimisticTransaction* txn, uint32_t column_family_id, const Endpoint &start_endp, @@ -222,10 +222,7 @@ class RangeLockMgr : void UnLock(const PessimisticTransaction* txn, const TransactionKeyMap* keys, Env* env) override ; - /* - Same as above, but *keys is guaranteed to hold all the locks obtained by - the transaction. - */ + // Release all locks the transaction is holding void UnLockAll(const PessimisticTransaction* txn, Env* env); void UnLock(PessimisticTransaction* txn, uint32_t column_family_id, const std::string& key, Env* env) override ; @@ -238,8 +235,7 @@ class RangeLockMgr : ~RangeLockMgr(); - int set_max_lock_memory(size_t max_lock_memory) override - { + int set_max_lock_memory(size_t max_lock_memory) override { return ltm_.set_max_lock_memory(max_lock_memory); } @@ -261,8 +257,10 @@ class RangeLockMgr : InstrumentedMutex ltree_map_mutex_; // Per-thread cache of ltree_map_. + // (uses the same approach as TransactionLockMgr::lock_maps_cache_) std::unique_ptr<ThreadLocalPtr> ltree_lookup_cache_; + // Get the lock tree which stores locks for Column Family with given cf_id toku::locktree *get_locktree_by_cfid(uint32_t cf_id); static int compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key);

1 0

[Commits] cd17512babf: Range Locking: test coverage for HA_READ_PREFIX_LAST_OR_PREV ranges.
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: cd17512babf2c1cd94b8ada4945a4993fd163e83 (fb-prod201801-231-gcd17512babf) parent(s): aca52c36332372a4aa71daf529bfb4383a6334d0 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 21:00:54 +0300 message: Range Locking: test coverage for HA_READ_PREFIX_LAST_OR_PREV ranges. --- mysql-test/suite/rocksdb/r/range_locking.result | 26 +++++++++++++++++ .../suite/rocksdb/r/range_locking_rev_cf.result | 26 +++++++++++++++++ mysql-test/suite/rocksdb/t/range_locking.inc | 22 ++++++++++++++ storage/rocksdb/ha_rocksdb.cc | 34 +++++++++++++++------- 4 files changed, 98 insertions(+), 10 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index a5e17f679e5..f8dad8c7b07 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -429,6 +429,32 @@ COLUMN_FAMILY_ID TRANSACTION_ID KEY mode $cf_id $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X $cf_id $trx_id 0000${indexnr}8000000280000009 X rollback; +# +# Test range locking for ranges with HA_READ_PREFIX_LAST_OR_PREV +# +begin; +# Should use range access with 2 keyparts and w/o filesort: +explain +select * from t1 +where pk1=4 and pk2 between 5 and 8 +order by pk1 desc, pk2 desc +for update; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range PRIMARY PRIMARY 8 NULL # Using where +select * from t1 +where pk1=4 and pk2 between 5 and 8 +order by pk1 desc, pk2 desc +for update; +pk1 pk2 a +4 8 48 +4 7 47 +4 6 46 +4 5 45 +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +$cf_id $trx_id 0000${indexnr}8000000480000005 - 0100${indexnr}8000000480000008 X +$cf_id $trx_id 0000${indexnr}8000000480000004 X +rollback; connection con1; rollback; disconnect con1; diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result index 6ab62f1208b..b07c654e7d1 100644 --- a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -389,6 +389,32 @@ COLUMN_FAMILY_ID TRANSACTION_ID KEY mode $cf_id $trx_id 0000${indexnr}8000000280000009 X $cf_id $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X rollback; +# +# Test range locking for ranges with HA_READ_PREFIX_LAST_OR_PREV +# +begin; +# Should use range access with 2 keyparts and w/o filesort: +explain +select * from t1 +where pk1=4 and pk2 between 5 and 8 +order by pk1 desc, pk2 desc +for update; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range PRIMARY PRIMARY 8 NULL # Using where +select * from t1 +where pk1=4 and pk2 between 5 and 8 +order by pk1 desc, pk2 desc +for update; +pk1 pk2 a +4 8 48 +4 7 47 +4 6 46 +4 5 45 +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +$cf_id $trx_id 0000${indexnr}8000000480000004 X +$cf_id $trx_id 0000${indexnr}8000000480000008 - 0100${indexnr}8000000480000005 X +rollback; connection con1; rollback; disconnect con1; diff --git a/mysql-test/suite/rocksdb/t/range_locking.inc b/mysql-test/suite/rocksdb/t/range_locking.inc index 265694436b1..1c5fb92b584 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.inc +++ b/mysql-test/suite/rocksdb/t/range_locking.inc @@ -458,6 +458,28 @@ let $select_from_is_rowlocks_current_trx_only=1; --source suite/rocksdb/include/select_from_is_rowlocks.inc rollback; +--echo # +--echo # Test range locking for ranges with HA_READ_PREFIX_LAST_OR_PREV +--echo # + +begin; +--echo # Should use range access with 2 keyparts and w/o filesort: +--replace_column 9 # +explain +select * from t1 +where pk1=4 and pk2 between 5 and 8 +order by pk1 desc, pk2 desc +for update; + +select * from t1 +where pk1=4 and pk2 between 5 and 8 +order by pk1 desc, pk2 desc +for update; + +let $select_from_is_rowlocks_current_trx_only=1; +--source suite/rocksdb/include/select_from_is_rowlocks.inc +rollback; + connection con1; rollback; disconnect con1; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 157e8f8bc16..9342ca84b04 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8160,12 +8160,14 @@ int ha_rocksdb::index_read_map(uchar *const buf, const uchar *const key, int ha_rocksdb::set_range_lock(Rdb_transaction *tx, const Rdb_key_def &kd, const enum ha_rkey_function &find_flag, - const rocksdb::Slice &slice, + const rocksdb::Slice &slice_arg, const key_range *const end_key) { rocksdb::Slice end_slice; uchar end_slice_buf[MAX_KEY_LENGTH]; bool start_has_inf_suffix = false, end_has_inf_suffix = false; + rocksdb::Slice slice(slice_arg); + if (m_lock_rows != RDB_LOCK_WRITE || !rocksdb_use_range_locking) { return 0; @@ -8200,18 +8202,30 @@ int ha_rocksdb::set_range_lock(Rdb_transaction *tx, end_has_inf_suffix= true; } else if (find_flag == HA_READ_PREFIX_LAST_OR_PREV) { - //psergey-todo: swap the bounds? so that the start bounds is the end? - //Testscase! - /* - We have made the kd.successor(m_sk_packed_tuple) call above. + We get here for queries like: - The slice is at least Rdb_key_def::INDEX_NUMBER_SIZE bytes long. + select * from t1 where pk1=const1 and pk2 between const2 and const3 + order by pk1 desc + for update + + assuming this uses an index on (pk1, pk2). + The slice has the right endpoint: {const1, const3} + the end_key has the left endpoint: {const1, const2}. */ - memcpy(end_slice_buf, slice.data(), slice.size()); - kd.successor(end_slice_buf, slice.size()); - end_slice= rocksdb::Slice((const char*)end_slice_buf, slice.size()); - start_has_inf_suffix= end_has_inf_suffix= false; + + // Move the right endpoint from slice to end_slice + end_slice= slice; + + // Pack the left endpoint and make "slice" point to it + uchar pack_buffer[MAX_KEY_LENGTH]; + uint end_slice_size= + kd.pack_index_tuple(table, pack_buffer, end_slice_buf, + end_key->key, end_key->keypart_map); + slice= rocksdb::Slice(reinterpret_cast<char *>(end_slice_buf), + end_slice_size); + start_has_inf_suffix= false; + end_has_inf_suffix= true; } else if (end_key) { // Known start range bounds: HA_READ_KEY_OR_NEXT, HA_READ_AFTER_KEY

1 0

[Commits] aca52c36332: Range Locking: information_schema.rocksdb_locks now has correct COLUMN_FAMILY_ID
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: aca52c36332372a4aa71daf529bfb4383a6334d0 (fb-prod201801-230-gaca52c36332) parent(s): 38b85e5ff2564d495d69857f17f4ada547cdb44f author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 20:24:38 +0300 message: Range Locking: information_schema.rocksdb_locks now has correct COLUMN_FAMILY_ID --- mysql-test/suite/rocksdb/r/range_locking.result | 6 +++--- mysql-test/suite/rocksdb/r/range_locking_rev_cf.result | 16 ++++++++-------- mysql-test/suite/rocksdb/t/range_locking.inc | 1 + rocksdb | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index e9f6d667931..a5e17f679e5 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -388,7 +388,7 @@ create table t1 ( pk1 int, pk2 int, a int, -primary key(pk1, pk2) comment '$pk_cf' +primary key(pk1, pk2) comment 'default' ) engine=rocksdb; insert into t1 select @@ -426,8 +426,8 @@ pk1 pk2 a 3 0 30 # select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc COLUMN_FAMILY_ID TRANSACTION_ID KEY mode -0 $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X -0 $trx_id 0000${indexnr}8000000280000009 X +$cf_id $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X +$cf_id $trx_id 0000${indexnr}8000000280000009 X rollback; connection con1; rollback; diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result index a52f35e1862..6ab62f1208b 100644 --- a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -93,12 +93,12 @@ pk a 10 10 # select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc COLUMN_FAMILY_ID TRANSACTION_ID KEY mode -0 $trx_id 0000${indexnr}8000000a X +$cf_id $trx_id 0000${indexnr}8000000a X delete from t1 where pk between 25 and 40; # select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc COLUMN_FAMILY_ID TRANSACTION_ID KEY mode -0 $trx_id 0000${indexnr}8000000a X -0 $trx_id 0000${indexnr}80000028 - 0100${indexnr}80000019 X +$cf_id $trx_id 0000${indexnr}8000000a X +$cf_id $trx_id 0000${indexnr}80000028 - 0100${indexnr}80000019 X rollback; begin; # The following will show a range lock on 2-9 and also a point lock on 10. @@ -107,8 +107,8 @@ select * from t1 where pk between 2 and 9 for update; pk a # select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc COLUMN_FAMILY_ID TRANSACTION_ID KEY mode -0 $trx_id 0000${indexnr}80000009 - 0100${indexnr}80000002 X -0 $trx_id 0000${indexnr}8000000a X +$cf_id $trx_id 0000${indexnr}80000009 - 0100${indexnr}80000002 X +$cf_id $trx_id 0000${indexnr}8000000a X rollback; drop table t1; # @@ -348,7 +348,7 @@ create table t1 ( pk1 int, pk2 int, a int, -primary key(pk1, pk2) comment '$pk_cf' +primary key(pk1, pk2) comment 'rev:cf1' ) engine=rocksdb; insert into t1 select @@ -386,8 +386,8 @@ pk1 pk2 a 3 0 30 # select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc COLUMN_FAMILY_ID TRANSACTION_ID KEY mode -0 $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X -0 $trx_id 0000${indexnr}8000000280000009 X +$cf_id $trx_id 0000${indexnr}8000000280000009 X +$cf_id $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X rollback; connection con1; rollback; diff --git a/mysql-test/suite/rocksdb/t/range_locking.inc b/mysql-test/suite/rocksdb/t/range_locking.inc index 866264176dc..265694436b1 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.inc +++ b/mysql-test/suite/rocksdb/t/range_locking.inc @@ -417,6 +417,7 @@ drop table t1; create table t0(a int) engine=rocksdb; insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +eval create table t1 ( pk1 int, pk2 int, diff --git a/rocksdb b/rocksdb index 293d6902411..bcc79236884 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 293d6902411748728fb48bf63e63ba121a963788 +Subproject commit bcc7923688446c58a21f4f0ab287b147f406abc2

1 0

[Commits] bcc792368: Range Locking: return accurate cf_id value in GetLockStatusData()
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: bcc7923688446c58a21f4f0ab287b147f406abc2 (v5.8-1040-gbcc792368) parent(s): 293d6902411748728fb48bf63e63ba121a963788 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 20:22:54 +0300 message: Range Locking: return accurate cf_id value in GetLockStatusData() This is possible now, as we don't put locks from all column families into one locktree anymore. --- utilities/transactions/transaction_lock_mgr.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index 95e819b3a..f837f6a5a 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -1293,11 +1293,10 @@ void push_into_lock_status_data(void* param, const DBT *left, BaseLockMgr::LockStatusData RangeLockMgr::GetLockStatusData() { LockStatusData data; - LOCK_PRINT_CONTEXT ctx = {&data, GetColumnFamilyID(my_txn_db_->DefaultColumnFamily()) }; - { InstrumentedMutexLock l(&ltree_map_mutex_); for (auto it : ltree_map_) { + LOCK_PRINT_CONTEXT ctx = {&data, it.first }; it.second->dump_locks((void*)&ctx, push_into_lock_status_data); } }

1 0

[Commits] 38b85e5ff25: Range Locking: add a testcase for HA_READ_PREFIX_LAST range
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 38b85e5ff2564d495d69857f17f4ada547cdb44f (fb-prod201801-229-g38b85e5ff25) parent(s): 0a80ecf45ff5973d4a01fb63e1f6fcdaf1ecf7e9 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 20:07:15 +0300 message: Range Locking: add a testcase for HA_READ_PREFIX_LAST range --- .../rocksdb/include/select_from_is_rowlocks.inc | 18 +++++-- mysql-test/suite/rocksdb/r/range_locking.result | 55 ++++++++++++++++++++++ .../suite/rocksdb/r/range_locking_rev_cf.result | 55 ++++++++++++++++++++++ mysql-test/suite/rocksdb/t/range_locking.inc | 52 ++++++++++++++++++++ storage/rocksdb/ha_rocksdb.cc | 15 ++++++ 5 files changed, 190 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc b/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc index 307b1e53552..61f3ee2c9f3 100644 --- a/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc +++ b/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc @@ -7,10 +7,18 @@ set @rtrx_id=(select transaction_id from information_schema.rocksdb_trx set @indexnr= (select lpad(hex(index_number),6,'0') from information_schema.rocksdb_ddl where table_name='t1' and index_name='PRIMARY'); -select - replace(column_family_id, @cf_id, "$cf_id") as COLUMN_FAMILY_ID, - replace(transaction_id, @rtrx_id, "$trx_id")as TRANSACTION_ID, - replace(`key`, @indexnr, '${indexnr}') as `KEY`, +let $extra_where = where 1; + +if ($select_from_is_rowlocks_current_trx_only) +{ + let $extra_where = where transaction_id=(select transaction_id from information_schema.rocksdb_trx where connection_id()=thread_id); +} + +eval select + replace(column_family_id, @cf_id, "\$cf_id") as COLUMN_FAMILY_ID, + replace(transaction_id, @rtrx_id, "\$trx_id")as TRANSACTION_ID, + replace(`key`, @indexnr, '\${indexnr}') as `KEY`, mode -from information_schema.rocksdb_locks; +from information_schema.rocksdb_locks $extra_where; + --enable_query_log diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index 5ab797b30d4..e9f6d667931 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -379,3 +379,58 @@ rollback; disconnect con1; connection default; drop table t1; +# +# Test range locking for ranges with HA_READ_PREFIX_LAST +# +create table t0(a int) engine=rocksdb; +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( +pk1 int, +pk2 int, +a int, +primary key(pk1, pk2) comment '$pk_cf' +) engine=rocksdb; +insert into t1 +select +A.a, B.a, A.a*10+B.a +from +t0 A, t0 B; +connect con1,localhost,root,,; +connection con1; +begin; +insert into t1 values (0x1112222,0x1112222,0); +connection default; +begin; +# Should use ref access w/o filesort: +explain +select * from t1 +where pk1=3 +order by pk1 desc, pk2 desc +for update; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref PRIMARY PRIMARY 4 const # Using where +select * from t1 +where pk1=3 +order by pk1 desc, pk2 desc +for update; +pk1 pk2 a +3 9 39 +3 8 38 +3 7 37 +3 6 36 +3 5 35 +3 4 34 +3 3 33 +3 2 32 +3 1 31 +3 0 30 +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +0 $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X +0 $trx_id 0000${indexnr}8000000280000009 X +rollback; +connection con1; +rollback; +disconnect con1; +connection default; +drop table t0, t1; diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result index 073ebd3afa8..a52f35e1862 100644 --- a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -339,3 +339,58 @@ rollback; disconnect con1; connection default; drop table t1; +# +# Test range locking for ranges with HA_READ_PREFIX_LAST +# +create table t0(a int) engine=rocksdb; +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 ( +pk1 int, +pk2 int, +a int, +primary key(pk1, pk2) comment '$pk_cf' +) engine=rocksdb; +insert into t1 +select +A.a, B.a, A.a*10+B.a +from +t0 A, t0 B; +connect con1,localhost,root,,; +connection con1; +begin; +insert into t1 values (0x1112222,0x1112222,0); +connection default; +begin; +# Should use ref access w/o filesort: +explain +select * from t1 +where pk1=3 +order by pk1 desc, pk2 desc +for update; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref PRIMARY PRIMARY 4 const # Using where +select * from t1 +where pk1=3 +order by pk1 desc, pk2 desc +for update; +pk1 pk2 a +3 9 39 +3 8 38 +3 7 37 +3 6 36 +3 5 35 +3 4 34 +3 3 33 +3 2 32 +3 1 31 +3 0 30 +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +0 $trx_id 0000${indexnr}80000003 - 0100${indexnr}80000003 X +0 $trx_id 0000${indexnr}8000000280000009 X +rollback; +connection con1; +rollback; +disconnect con1; +connection default; +drop table t0, t1; diff --git a/mysql-test/suite/rocksdb/t/range_locking.inc b/mysql-test/suite/rocksdb/t/range_locking.inc index db71f91c3b0..866264176dc 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.inc +++ b/mysql-test/suite/rocksdb/t/range_locking.inc @@ -411,3 +411,55 @@ disconnect con1; connection default; drop table t1; +--echo # +--echo # Test range locking for ranges with HA_READ_PREFIX_LAST +--echo # +create table t0(a int) engine=rocksdb; +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t1 ( + pk1 int, + pk2 int, + a int, + primary key(pk1, pk2) comment '$pk_cf' +) engine=rocksdb; + +insert into t1 +select + A.a, B.a, A.a*10+B.a +from + t0 A, t0 B; + + +# Get a lock in another connection so that the primary transaction is not using +# STO optimization, and its locks can be seen in I_S.rocksdb_locks +--connect (con1,localhost,root,,) +connection con1; +begin; +insert into t1 values (0x1112222,0x1112222,0); + +connection default; +begin; +--echo # Should use ref access w/o filesort: +--replace_column 9 # +explain +select * from t1 +where pk1=3 +order by pk1 desc, pk2 desc +for update; + +select * from t1 +where pk1=3 +order by pk1 desc, pk2 desc +for update; + +let $select_from_is_rowlocks_current_trx_only=1; +--source suite/rocksdb/include/select_from_is_rowlocks.inc +rollback; + +connection con1; +rollback; +disconnect con1; +connection default; + +drop table t0, t1; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 0c4dea535ca..157e8f8bc16 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8185,6 +8185,21 @@ int ha_rocksdb::set_range_lock(Rdb_transaction *tx, end_slice= slice; } else if (find_flag == HA_READ_PREFIX_LAST) { + /* + We get here for queries like: + + select * from t1 where pk1=const order by pk1 desc for update + + assuming this uses an index on (pk1, ...) + We get end_key=nullptr. + + The range to lock is the same as with HA_READ_KEY_EXACT above. + */ + end_slice= slice; + start_has_inf_suffix= false; + end_has_inf_suffix= true; + } + else if (find_flag == HA_READ_PREFIX_LAST_OR_PREV) { //psergey-todo: swap the bounds? so that the start bounds is the end? //Testscase!

1 0

[Commits] 0a80ecf45ff: Range Locking: small fixes
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 0a80ecf45ff5973d4a01fb63e1f6fcdaf1ecf7e9 (fb-prod201801-228-g0a80ecf45ff) parent(s): c523d5bd0e776a88495405dc9ba305da9fe17600 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 18:46:43 +0300 message: Range Locking: small fixes - Update rocksdb ( Range Locking : minor fixes) - Remove rocksdb.range_locking dependencies from indexnr, transaction nr, cf_id and other transient numbers. --- .../rocksdb/include/select_from_is_rowlocks.inc | 16 +++++++++++++ mysql-test/suite/rocksdb/r/range_locking.result | 26 +++++++++++----------- .../suite/rocksdb/r/range_locking_rev_cf.result | 26 +++++++++++----------- mysql-test/suite/rocksdb/t/range_locking.inc | 13 +++++------ rocksdb | 2 +- 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc b/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc new file mode 100644 index 00000000000..307b1e53552 --- /dev/null +++ b/mysql-test/suite/rocksdb/include/select_from_is_rowlocks.inc @@ -0,0 +1,16 @@ +--echo # select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +--disable_query_log +set @cf_id=(select column_family from information_schema.rocksdb_ddl + where table_name='t1' and index_name='PRIMARY'); +set @rtrx_id=(select transaction_id from information_schema.rocksdb_trx + where thread_id=connection_id()); +set @indexnr= (select lpad(hex(index_number),6,'0') from information_schema.rocksdb_ddl + where table_name='t1' and index_name='PRIMARY'); + +select + replace(column_family_id, @cf_id, "$cf_id") as COLUMN_FAMILY_ID, + replace(transaction_id, @rtrx_id, "$trx_id")as TRANSACTION_ID, + replace(`key`, @indexnr, '${indexnr}') as `KEY`, + mode +from information_schema.rocksdb_locks; +--enable_query_log diff --git a/mysql-test/suite/rocksdb/r/range_locking.result b/mysql-test/suite/rocksdb/r/range_locking.result index 0518260f53a..5ab797b30d4 100644 --- a/mysql-test/suite/rocksdb/r/range_locking.result +++ b/mysql-test/suite/rocksdb/r/range_locking.result @@ -23,7 +23,6 @@ ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on connection con1; rollback; ## Test: check that range lock inhibits another range lock -## (It should not, but currently it does) connection con1; begin; select * from t1 where pk between 5 and 25 for update; @@ -53,7 +52,8 @@ pk a rollback; connection con1; rollback; -## Test that locks are not released when a statement in the transaction +## Test that locks are not released when a statement inside +## a transaction is rolled back create table t2 ( pk int, a int, @@ -91,24 +91,24 @@ begin; select * from t1 where pk=10 for update; pk a 10 10 -select * from information_schema.rocksdb_locks; -COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 00000001078000000a X +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +$cf_id $trx_id 0000${indexnr}8000000a X delete from t1 where pk between 25 and 40; -select * from information_schema.rocksdb_locks; -COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 000000010780000019 - 010000010780000028 X -0 15 00000001078000000a X +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +$cf_id $trx_id 0000${indexnr}80000019 - 0100${indexnr}80000028 X +$cf_id $trx_id 0000${indexnr}8000000a X rollback; begin; # The following will show a range lock on 2-9 and also a point lock on 10. # This is how things currently work. select * from t1 where pk between 2 and 9 for update; pk a -select * from information_schema.rocksdb_locks; -COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 16 00000001078000000a X -0 16 000000010780000002 - 010000010780000009 X +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +$cf_id $trx_id 0000${indexnr}8000000a X +$cf_id $trx_id 0000${indexnr}80000002 - 0100${indexnr}80000009 X rollback; drop table t1; # diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result index c2f8064c6b7..073ebd3afa8 100644 --- a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -23,7 +23,6 @@ ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on connection con1; rollback; ## Test: check that range lock inhibits another range lock -## (It should not, but currently it does) connection con1; begin; select * from t1 where pk between 5 and 25 for update; @@ -53,7 +52,8 @@ pk a rollback; connection con1; rollback; -## Test that locks are not released when a statement in the transaction +## Test that locks are not released when a statement inside +## a transaction is rolled back create table t2 ( pk int, a int, @@ -91,24 +91,24 @@ begin; select * from t1 where pk=10 for update; pk a 10 10 -select * from information_schema.rocksdb_locks; -COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 00000001078000000a X +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +0 $trx_id 0000${indexnr}8000000a X delete from t1 where pk between 25 and 40; -select * from information_schema.rocksdb_locks; -COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 00000001078000000a X -0 15 000000010780000028 - 010000010780000019 X +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +0 $trx_id 0000${indexnr}8000000a X +0 $trx_id 0000${indexnr}80000028 - 0100${indexnr}80000019 X rollback; begin; # The following will show a range lock on 2-9 and also a point lock on 10. # This is how things currently work. select * from t1 where pk between 2 and 9 for update; pk a -select * from information_schema.rocksdb_locks; -COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 16 000000010780000009 - 010000010780000002 X -0 16 00000001078000000a X +# select * from information_schema.rocksdb_locks; # With replacements by select_from_is_rowlocks.inc +COLUMN_FAMILY_ID TRANSACTION_ID KEY mode +0 $trx_id 0000${indexnr}80000009 - 0100${indexnr}80000002 X +0 $trx_id 0000${indexnr}8000000a X rollback; drop table t1; # diff --git a/mysql-test/suite/rocksdb/t/range_locking.inc b/mysql-test/suite/rocksdb/t/range_locking.inc index 9c826811695..db71f91c3b0 100644 --- a/mysql-test/suite/rocksdb/t/range_locking.inc +++ b/mysql-test/suite/rocksdb/t/range_locking.inc @@ -10,8 +10,6 @@ show variables like 'rocksdb_use_range_locking'; -# Test 1: see that ranges are indeed lockied. - eval create table t1 ( pk int, a int, @@ -37,7 +35,6 @@ connection con1; rollback; --echo ## Test: check that range lock inhibits another range lock ---echo ## (It should not, but currently it does) connection con1; begin; select * from t1 where pk between 5 and 25 for update; @@ -65,7 +62,8 @@ rollback; connection con1; rollback; ---echo ## Test that locks are not released when a statement in the transaction +--echo ## Test that locks are not released when a statement inside +--echo ## a transaction is rolled back eval create table t2 ( pk int, @@ -114,17 +112,18 @@ insert into t1 values begin; select * from t1 where pk=10 for update; -select * from information_schema.rocksdb_locks; +--source suite/rocksdb/include/select_from_is_rowlocks.inc delete from t1 where pk between 25 and 40; -select * from information_schema.rocksdb_locks; + +--source suite/rocksdb/include/select_from_is_rowlocks.inc rollback; begin; --echo # The following will show a range lock on 2-9 and also a point lock on 10. --echo # This is how things currently work. select * from t1 where pk between 2 and 9 for update; -select * from information_schema.rocksdb_locks; +--source suite/rocksdb/include/select_from_is_rowlocks.inc rollback; drop table t1; diff --git a/rocksdb b/rocksdb index d334cf12818..293d6902411 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit d334cf12818bfd2d44dc06c7bbe78c3a18bca543 +Subproject commit 293d6902411748728fb48bf63e63ba121a963788

1 0

[Commits] ec81406: MDEV-16060 - InnoDB: Failing assertion: ut_strcmp(index->name, key->name)
by Sergey Vojtovich 29 Apr '19

29 Apr '19

revision-id: ec81406ea5c63a07c3810252a97f9ac9ecf6b85c (mariadb-10.1.38-92-gec81406) parent(s): d0116e10a5da52503a89a413e481996ce3f65e63 author: Sergey Vojtovich committer: Sergey Vojtovich timestamp: 2019-04-29 19:32:42 +0400 message: MDEV-16060 - InnoDB: Failing assertion: ut_strcmp(index->name, key->name) A sequel to 9180e86 and 149b754. ALTER TABLE ... ADD FOREIGN KEY may crash if parent table is updated concurrently. Block FK parent table updates even earlier, before intermediate child table is created. Use proper charset info for my_casedn_str() and don't update original identifiers so that lower_cast_table_names == 2 is honoured. --- mysql-test/suite/innodb/r/foreign-keys.result | 23 +++++++ mysql-test/suite/innodb/t/foreign-keys.test | 30 +++++++++ sql/sql_table.cc | 91 +++++++++++++-------------- 3 files changed, 98 insertions(+), 46 deletions(-) diff --git a/mysql-test/suite/innodb/r/foreign-keys.result b/mysql-test/suite/innodb/r/foreign-keys.result index 6852852..82ca9c3 100644 --- a/mysql-test/suite/innodb/r/foreign-keys.result +++ b/mysql-test/suite/innodb/r/foreign-keys.result @@ -100,3 +100,26 @@ CREATE TABLE t2 (b INT, KEY(b)) ENGINE=InnoDB; INSERT INTO t2 VALUES(2); ALTER TABLE t2 ADD FOREIGN KEY(b) REFERENCES t1(a), LOCK=EXCLUSIVE; DROP TABLE t2, t1; +# +# MDEV-16060 - InnoDB: Failing assertion: ut_strcmp(index->name, key->name) +# +CREATE TABLE t1 (`pk` INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t2 LIKE t1; +FLUSH TABLES; +connect con1, localhost, root; +SET debug_sync='alter_table_intermediate_table_created SIGNAL ready WAIT_FOR go'; +ALTER TABLE t1 ADD FOREIGN KEY(pk) REFERENCES t2(pk) ON UPDATE CASCADE; +connection default; +SET debug_sync='now WAIT_FOR ready'; +SET lock_wait_timeout=1; +UPDATE t2 SET pk=10 WHERE pk=1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +FLUSH TABLE t2; +SET debug_sync='now SIGNAL go'; +SET debug_sync='reset'; +connection con1; +disconnect con1; +connection default; +SHOW OPEN TABLES; +Database Table In_use Name_locked +DROP TABLE t1, t2; diff --git a/mysql-test/suite/innodb/t/foreign-keys.test b/mysql-test/suite/innodb/t/foreign-keys.test index ced44a8..d41c8b2 100644 --- a/mysql-test/suite/innodb/t/foreign-keys.test +++ b/mysql-test/suite/innodb/t/foreign-keys.test @@ -126,3 +126,33 @@ CREATE TABLE t2 (b INT, KEY(b)) ENGINE=InnoDB; INSERT INTO t2 VALUES(2); ALTER TABLE t2 ADD FOREIGN KEY(b) REFERENCES t1(a), LOCK=EXCLUSIVE; DROP TABLE t2, t1; + + +--echo # +--echo # MDEV-16060 - InnoDB: Failing assertion: ut_strcmp(index->name, key->name) +--echo # +CREATE TABLE t1 (`pk` INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t2 LIKE t1; +FLUSH TABLES; + +connect con1, localhost, root; +SET debug_sync='alter_table_intermediate_table_created SIGNAL ready WAIT_FOR go'; +send ALTER TABLE t1 ADD FOREIGN KEY(pk) REFERENCES t2(pk) ON UPDATE CASCADE; + +connection default; +SET debug_sync='now WAIT_FOR ready'; +SET lock_wait_timeout=1; # change to 0 in 10.3 +--error ER_LOCK_WAIT_TIMEOUT +UPDATE t2 SET pk=10 WHERE pk=1; +FLUSH TABLE t2; +SET debug_sync='now SIGNAL go'; +SET debug_sync='reset'; + +connection con1; +reap; + +# Cleanup +disconnect con1; +connection default; +SHOW OPEN TABLES; +DROP TABLE t1, t2; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 57333e7..463aab9 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -8242,6 +8242,50 @@ static bool fk_prepare_copy_alter_table(THD *thd, TABLE *table, } } + /* + Normally, an attempt to modify an FK parent table will cause + FK children to be prelocked, so the table-being-altered cannot + be modified by a cascade FK action, because ALTER holds a lock + and prelocking will wait. + + But if a new FK is being added by this very ALTER, then the target + table is not locked yet (it's a temporary table). So, we have to + lock FK parents explicitly. + */ + if (alter_info->flags & Alter_info::ADD_FOREIGN_KEY) + { + List_iterator<Key> fk_list_it(alter_info->key_list); + + while (Key *key= fk_list_it++) + { + if (key->type != Key::FOREIGN_KEY) + continue; + + Foreign_key *fk= static_cast<Foreign_key*>(key); + char dbuf[NAME_LEN]; + char tbuf[NAME_LEN]; + const char *ref_db= fk->ref_db.str ? fk->ref_db.str : alter_ctx->new_db; + const char *ref_table= fk->ref_table.str; + MDL_request mdl_request; + + if (lower_case_table_names) + { + strmake_buf(dbuf, ref_db); + my_casedn_str(system_charset_info, dbuf); + strmake_buf(tbuf, ref_table); + my_casedn_str(system_charset_info, tbuf); + ref_db= dbuf; + ref_table= tbuf; + } + + mdl_request.init(MDL_key::TABLE, ref_db, ref_table, MDL_SHARED_NO_WRITE, + MDL_TRANSACTION); + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + DBUG_RETURN(true); + } + } + DBUG_RETURN(false); } @@ -9093,6 +9137,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, /* Mark that we have created table in storage engine. */ no_ha_table= false; + DEBUG_SYNC(thd, "alter_table_intermediate_table_created"); if (create_info->tmp_table()) { @@ -9126,52 +9171,6 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, true, true); if (!new_table) goto err_new_table_cleanup; - - /* - Normally, an attempt to modify an FK parent table will cause - FK children to be prelocked, so the table-being-altered cannot - be modified by a cascade FK action, because ALTER holds a lock - and prelocking will wait. - - But if a new FK is being added by this very ALTER, then the target - table is not locked yet (it's a temporary table). So, we have to - lock FK parents explicitly. - */ - if (alter_info->flags & Alter_info::ADD_FOREIGN_KEY) - { - List <FOREIGN_KEY_INFO> fk_list; - List_iterator<FOREIGN_KEY_INFO> fk_list_it(fk_list); - FOREIGN_KEY_INFO *fk; - - /* tables_opened can be > 1 only for MERGE tables */ - DBUG_ASSERT(tables_opened == 1); - DBUG_ASSERT(&table_list->next_global == thd->lex->query_tables_last); - - new_table->file->get_foreign_key_list(thd, &fk_list); - while ((fk= fk_list_it++)) - { - MDL_request mdl_request; - - if (lower_case_table_names) - { - char buf[NAME_LEN]; - uint len; - strmake_buf(buf, fk->referenced_db->str); - len = my_casedn_str(files_charset_info, buf); - thd->make_lex_string(fk->referenced_db, buf, len); - strmake_buf(buf, fk->referenced_table->str); - len = my_casedn_str(files_charset_info, buf); - thd->make_lex_string(fk->referenced_table, buf, len); - } - - mdl_request.init(MDL_key::TABLE, - fk->referenced_db->str, fk->referenced_table->str, - MDL_SHARED_NO_WRITE, MDL_TRANSACTION); - if (thd->mdl_context.acquire_lock(&mdl_request, - thd->variables.lock_wait_timeout)) - goto err_new_table_cleanup; - } - } } /* Note: In case of MERGE table, we do not attach children. We do not

1 0

[Commits] 293d69024: Range Locking : minor fixes
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: 293d6902411748728fb48bf63e63ba121a963788 (v5.8-1039-g293d69024) parent(s): d334cf12818bfd2d44dc06c7bbe78c3a18bca543 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 18:11:02 +0300 message: Range Locking : minor fixes - TransactionDB::init: Unused variable - RangeLockList::append: always append the new range to the list (even if we've just created the transaction) - PessimisticTransactionDB::TryRangeLock: safer value for currently-unused parameter --- utilities/transactions/pessimistic_transaction_db.cc | 2 +- utilities/transactions/transaction_lock_mgr.cc | 4 +--- utilities/transactions/transaction_lock_mgr.h | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/utilities/transactions/pessimistic_transaction_db.cc b/utilities/transactions/pessimistic_transaction_db.cc index e3ce435af..14b82de00 100644 --- a/utilities/transactions/pessimistic_transaction_db.cc +++ b/utilities/transactions/pessimistic_transaction_db.cc @@ -405,7 +405,7 @@ PessimisticTransactionDB::TryRangeLock(PessimisticTransaction *txn, const Endpoint& end_endp) { if (range_lock_mgr_) { return range_lock_mgr_->TryRangeLock(txn, cfh_id, start_endp, - end_endp, /*exclusive=*/false); + end_endp, /*exclusive=*/true); } else return Status::NotSupported(); diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index c375b24f2..95e819b3a 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -798,12 +798,10 @@ public: auto it= buffers_.find(cf_id); if (it == buffers_.end()) { // create a new one - //it->second.create(); it= buffers_.emplace(cf_id, std::shared_ptr<toku::range_buffer>(new toku::range_buffer())).first; it->second->create(); } - else - it->second->append(left_key, right_key); + it->second->append(left_key, right_key); } std::unordered_map<uint32_t, std::shared_ptr<toku::range_buffer>> buffers_; diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index d35bf5935..1f5dbc634 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -78,7 +78,7 @@ class BaseLockMgr { virtual std::vector<DeadlockPath> GetDeadlockInfoBuffer()= 0; // TransactionDB will call this at start - virtual void init(TransactionDB *db_arg) {}; + virtual void init(TransactionDB*) {}; virtual ~BaseLockMgr(){} using LockStatusData = std::unordered_multimap<uint32_t, KeyLockInfo>;

1 0

[Commits] c523d5bd0e7: Range Locking: let RocksDB use user-provided comparator
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: c523d5bd0e776a88495405dc9ba305da9fe17600 (fb-prod201801-227-gc523d5bd0e7) parent(s): 70e12615732d658939972b24da162de8d57086f0 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 16:54:38 +0300 message: Range Locking: let RocksDB use user-provided comparator The order of endpoints in GetRangeLock() call must match the used ordering --- mysql-test/suite/rocksdb/r/range_locking_rev_cf.result | 4 ++-- rocksdb | 2 +- storage/rocksdb/ha_rocksdb.cc | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result index 444c51162cd..c2f8064c6b7 100644 --- a/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result +++ b/mysql-test/suite/rocksdb/r/range_locking_rev_cf.result @@ -97,8 +97,8 @@ COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE delete from t1 where pk between 25 and 40; select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE -0 15 000000010780000019 - 010000010780000028 X 0 15 00000001078000000a X +0 15 000000010780000028 - 010000010780000019 X rollback; begin; # The following will show a range lock on 2-9 and also a point lock on 10. @@ -107,8 +107,8 @@ select * from t1 where pk between 2 and 9 for update; pk a select * from information_schema.rocksdb_locks; COLUMN_FAMILY_ID TRANSACTION_ID KEY MODE +0 16 000000010780000009 - 010000010780000002 X 0 16 00000001078000000a X -0 16 000000010780000002 - 010000010780000009 X rollback; drop table t1; # diff --git a/rocksdb b/rocksdb index 9af13f0e966..d334cf12818 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 9af13f0e966171ef8688484b2da7fa576a1b4574 +Subproject commit d334cf12818bfd2d44dc06c7bbe78c3a18bca543 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index dea7fb9e53f..0c4dea535ca 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8245,9 +8245,18 @@ int ha_rocksdb::set_range_lock(Rdb_transaction *tx, end_has_inf_suffix= true; } - auto s= tx->lock_range(kd.get_cf(), - rocksdb::Endpoint(slice, start_has_inf_suffix), - rocksdb::Endpoint(end_slice, end_has_inf_suffix)); + rocksdb::Endpoint start_endp; + rocksdb::Endpoint end_endp; + if (kd.m_is_reverse_cf) { + // Flip the endpoints + start_endp =rocksdb::Endpoint(end_slice, !end_has_inf_suffix); + end_endp = rocksdb::Endpoint(slice, !start_has_inf_suffix); + } else { + start_endp= rocksdb::Endpoint(slice, start_has_inf_suffix); + end_endp= rocksdb::Endpoint(end_slice, end_has_inf_suffix); + } + + auto s= tx->lock_range(kd.get_cf(), start_endp, end_endp); if (!s.ok()) { return (tx->set_status_error(table->in_use, s, kd, m_tbl_def, m_table_handler));

1 0

[Commits] d334cf128: Range Locking: make the range tree code invoke the user-provided comparator
by Sergei Petrunia 29 Apr '19

29 Apr '19

revision-id: d334cf12818bfd2d44dc06c7bbe78c3a18bca543 (v5.8-1038-gd334cf128) parent(s): 9af13f0e966171ef8688484b2da7fa576a1b4574 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-04-29 16:53:06 +0300 message: Range Locking: make the range tree code invoke the user-provided comparator --- include/rocksdb/utilities/transaction.h | 8 +++++ utilities/transactions/transaction_lock_mgr.cc | 49 ++++++++++---------------- utilities/transactions/transaction_lock_mgr.h | 8 +---- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/include/rocksdb/utilities/transaction.h b/include/rocksdb/utilities/transaction.h index 20f9b1bbd..b0f1308ed 100644 --- a/include/rocksdb/utilities/transaction.h +++ b/include/rocksdb/utilities/transaction.h @@ -80,6 +80,12 @@ using TransactionID = uint64_t; class Endpoint { public: Slice slice; + + /* + true : the key has an "infinity" suffix. A suffix that would compare as + greater than any other suffix + false : otherwise + */ bool inf_suffix; Endpoint(const Slice &slice_arg, bool inf_suffix_arg=false) : @@ -90,6 +96,8 @@ class Endpoint { Endpoint(const char* s, size_t size, bool inf_suffix_arg=false) : slice(s, size), inf_suffix(inf_suffix_arg) {} + + Endpoint() : inf_suffix(false) {} }; // Provides notification to the caller of SetSnapshotOnNextOperation when diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index c98803acb..c375b24f2 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -813,11 +813,12 @@ public: bool releasing_locks_; }; +static const char SUFFIX_INFIMUM= 0x0; +static const char SUFFIX_SUPREMUM= 0x1; + static void serialize_endpoint(const Endpoint &endp, std::string *buf) { - const char SUFFIX_INF= 0x0; - const char SUFFIX_SUP= 0x1; - buf->push_back(endp.inf_suffix ? SUFFIX_SUP : SUFFIX_INF); + buf->push_back(endp.inf_suffix ? SUFFIX_SUPREMUM : SUFFIX_INFIMUM); buf->append(endp.slice.data(), endp.slice.size()); } @@ -1044,11 +1045,6 @@ void RangeLockMgr::UnLockAll(const PessimisticTransaction* txn, Env*) { int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key) { - //RangeLockMgr* mgr= (RangeLockMgr*) arg; - // TODO: this should compare endpoints using the user-provided comparator + - // endpoint encoding. - // (just use one from any column family) - const char *a= (const char*)a_key->data; const char *b= (const char*)b_key->data; @@ -1057,16 +1053,16 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, size_t min_len= std::min(a_len, b_len); - //compare the values. Skip the first byte as it is the endpoint signifier - - //TODO: use the upper layer' comparison function. - int res= memcmp(a+1, b+1, min_len-1); + // Compare the values. The first byte encodes the endpoint type, its value + // is either SUFFIX_INFIMUM or SUFFIX_SUPREMUM. + Comparator *cmp = (Comparator*) arg; + int res= cmp->Compare(Slice(a+1, min_len-1), Slice(b+1, min_len-1)); if (!res) { if (b_len > min_len) { // a is shorter; - if (a[0] == 0) + if (a[0] == SUFFIX_INFIMUM) return -1; //"a is smaller" else { @@ -1077,7 +1073,7 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, else if (a_len > min_len) { // the opposite of the above: b is shorter. - if (b[0] == 0) + if (b[0] == SUFFIX_INFIMUM) return 1; //"b is smaller" else { @@ -1100,21 +1096,10 @@ int RangeLockMgr::compare_dbt_endpoints(__toku_db*, void *arg, return res; } -int RangeLockMgr::compare_dbt_endpoints_rev(__toku_db* db, void *arg, - const DBT *a_key, - const DBT *b_key) { - return -compare_dbt_endpoints(db, arg, a_key, b_key); -} - - RangeLockMgr::RangeLockMgr(std::shared_ptr<TransactionDBMutexFactory> mutex_factory) : mutex_factory_(mutex_factory), ltree_lookup_cache_(new ThreadLocalPtr(nullptr)) { - ltm_.create(on_create, on_destroy, on_escalate, NULL, mutex_factory_); - - fw_cmp_.create(compare_dbt_endpoints, (void*)this, NULL); - bw_cmp_.create(compare_dbt_endpoints, (void*)this, NULL); } @@ -1168,8 +1153,6 @@ RangeLockMgr::~RangeLockMgr() { ltm_.release_lt(it.second); } ltm_.destroy(); - fw_cmp_.destroy(); - bw_cmp_.destroy(); } uint64_t RangeLockMgr::get_escalation_count() { @@ -1199,18 +1182,24 @@ void RangeLockMgr::AddColumnFamily(const ColumnFamilyHandle *cfh) { InstrumentedMutexLock l(&ltree_map_mutex_); if (ltree_map_.find(column_family_id) == ltree_map_.end()) { DICTIONARY_ID dict_id = { .dictid = column_family_id }; - toku::comparator *ltree_cmp; // "RocksDB_SE_v3.10" // BytewiseComparator() ,ReverseBytewiseComparator() + /* + toku::comparator *ltree_cmp; if (!strcmp(cfh->GetComparator()->Name(), "RocksDB_SE_v3.10")) ltree_cmp = &fw_cmp_; else if (!strcmp(cfh->GetComparator()->Name(),"rev:RocksDB_SE_v3.10")) - ltree_cmp = &fw_cmp_; // temporary: use the same ordering + ltree_cmp = &bw_cmp_; else { assert(false); ltree_cmp= nullptr; } - toku::locktree *ltree= ltm_.get_lt(dict_id, *ltree_cmp, + */ + toku::comparator cmp; + cmp.create(compare_dbt_endpoints, (void*)cfh->GetComparator(), NULL); + toku::locktree *ltree= ltm_.get_lt(dict_id, cmp, /* on_create_extra*/nullptr); + // This is ok to because get_lt has copied the comparator: + cmp.destroy(); ltree_map_.emplace(column_family_id, ltree); } else { // column_family already exists in lock map diff --git a/utilities/transactions/transaction_lock_mgr.h b/utilities/transactions/transaction_lock_mgr.h index e41a4961a..d35bf5935 100644 --- a/utilities/transactions/transaction_lock_mgr.h +++ b/utilities/transactions/transaction_lock_mgr.h @@ -250,9 +250,6 @@ class RangeLockMgr : private: toku::locktree_manager ltm_; - toku::comparator fw_cmp_; - toku::comparator bw_cmp_; - TransactionDB* my_txn_db_; std::shared_ptr<TransactionDBMutexFactory> mutex_factory_; @@ -269,8 +266,7 @@ class RangeLockMgr : toku::locktree *get_locktree_by_cfid(uint32_t cf_id); static int compare_dbt_endpoints(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); - static int compare_dbt_endpoints_rev(__toku_db*, void *arg, const DBT *a_key, const DBT *b_key); - + // Callbacks static int on_create(locktree*, void*) { return 0; /* no error */ } static void on_destroy(locktree*) {} @@ -279,7 +275,5 @@ class RangeLockMgr : }; - - } // namespace rocksdb #endif // ROCKSDB_LITE

1 0