[Commits] 25b5345: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge

revision-id: 25b5345aff6333f1dfa28f5d95bc363d0bc1f535 parent(s): 55163ba1bdb1a05daadc66c41c959994231b361c faa4d8f8c6c12c09541c4733773ac700c5e643cf committer: Sergei Petrunia branch nick: 10.2-mariarocks timestamp: 2018-08-28 12:49:50 +0300 message: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge Conflicts: storage/rocksdb/CMakeLists.txt storage/rocksdb/ha_rocksdb.cc storage/rocksdb/ha_rocksdb.h storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result storage/rocksdb/mysql-test/rocksdb/r/transaction.result storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result storage/rocksdb/mysql-test/rocksdb/r/validate_datadic.result storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test storage/rocksdb/mysql-test/rocksdb/t/bloomfilter5-master.opt storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test storage/rocksdb/mysql-test/rocksdb/t/collation.test storage/rocksdb/mysql-test/rocksdb/t/information_schema.test storage/rocksdb/mysql-test/rocksdb/t/prefix_extractor_override.test storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test storage/rocksdb/mysql-test/rocksdb/t/rocksdb_cf_options.test storage/rocksdb/mysql-test/rocksdb/t/rocksdb_checksums.test storage/rocksdb/mysql-test/rocksdb/t/transaction.test storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test storage/rocksdb/rdb_i_s.cc storage/rocksdb/rdb_i_s.h storage/rocksdb/CMakeLists.txt | 8 +- storage/rocksdb/ha_rocksdb.cc | 1219 +++++++++++++++----- storage/rocksdb/ha_rocksdb.h | 53 +- storage/rocksdb/ha_rocksdb_proto.h | 7 +- storage/rocksdb/mysql-test/rocksdb/combinations | 6 + .../rocksdb/include/have_write_committed.inc | 3 + .../mysql-test/rocksdb/r/add_index_inplace.result | 24 + .../r/add_index_inplace_sstfilewriter.result | 2 +- .../rocksdb/r/allow_no_primary_key.result | 6 +- .../mysql-test/rocksdb/r/autoinc_debug.result | 40 +- .../mysql-test/rocksdb/r/autoinc_vars.result | 18 + .../mysql-test/rocksdb/r/bloomfilter5.result | 4 +- .../rocksdb/r/bloomfilter_bulk_load.result | 15 + .../mysql-test/rocksdb/r/bulk_load_sk.result | 229 ++++ .../mysql-test/rocksdb/r/cardinality.result | 17 +- .../rocksdb/mysql-test/rocksdb/r/collation.result | 20 +- .../rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result | 21 + .../rocksdb/r/create_no_primary_key_table.result | 38 + .../mysql-test/rocksdb/r/ddl_high_priority.result | 71 +- .../mysql-test/rocksdb/r/deadlock_tracking.result | 57 + .../rocksdb/mysql-test/rocksdb/r/drop_table.result | 11 + .../mysql-test/rocksdb/r/explicit_snapshot.result | 265 +++++ .../mysql-test/rocksdb/r/i_s_deadlock.result | 55 +- .../mysql-test/rocksdb/r/index_file_map.result | 3 + .../rocksdb/r/index_merge_rocksdb2.result | 52 +- .../mysql-test/rocksdb/r/information_schema.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/lock.result | 15 + .../rocksdb/mysql-test/rocksdb/r/mysqldump.result | 5 +- .../r/percona_nonflushing_analyze_debug.result | 19 + .../rocksdb/r/prefix_extractor_override.result | 7 +- .../mysql-test/rocksdb/r/read_only_tx.result | 2 +- .../rocksdb/mysql-test/rocksdb/r/rocksdb.result | 24 +- .../mysql-test/rocksdb/r/rocksdb_checksums.result | 6 +- .../rocksdb/r/rocksdb_deadlock_detect_rc.result | 16 +- .../rocksdb/r/rocksdb_deadlock_detect_rr.result | 16 +- .../rocksdb/r/secondary_key_update_lock.result | 11 + .../mysql-test/rocksdb/r/show_engine.result | 53 + .../mysql-test/rocksdb/r/transaction.result | 17 + .../mysql-test/rocksdb/r/trx_info_rpl.result | 2 +- storage/rocksdb/mysql-test/rocksdb/r/update.result | 8 + .../mysql-test/rocksdb/r/use_direct_reads.result | 18 + .../mysql-test/rocksdb/t/add_index_inplace.test | 18 + .../rocksdb/t/add_index_inplace_crash.test | 1 + .../rocksdb/t/add_index_inplace_sstfilewriter.test | 2 +- .../mysql-test/rocksdb/t/autoinc_debug.test | 17 +- .../rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 23 + .../mysql-test/rocksdb/t/bloomfilter5-master.opt | 4 +- .../rocksdb/t/bloomfilter_bulk_load-master.opt | 2 + .../rocksdb/t/bloomfilter_bulk_load.test | 35 + .../rocksdb/mysql-test/rocksdb/t/bulk_load_sk.test | 119 ++ .../rocksdb/mysql-test/rocksdb/t/cardinality.test | 21 +- .../rocksdb/t/check_ignore_unknown_options.test | 3 + .../rocksdb/mysql-test/rocksdb/t/collation.test | 36 +- .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.cnf | 4 + .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.test | 87 ++ .../t/create_no_primary_key_table-master.opt | 1 + .../rocksdb/t/create_no_primary_key_table.test | 44 + .../mysql-test/rocksdb/t/deadlock_tracking.test | 76 +- .../rocksdb/mysql-test/rocksdb/t/drop_table.test | 27 + .../rocksdb/t/explicit_snapshot-master.opt | 1 + .../mysql-test/rocksdb/t/explicit_snapshot.test | 263 +++++ .../rocksdb/mysql-test/rocksdb/t/i_s_deadlock.test | 14 +- .../mysql-test/rocksdb/t/index_file_map.test | 3 + .../rocksdb/t/index_merge_rocksdb2-master.opt | 2 +- .../mysql-test/rocksdb/t/information_schema.test | 1 + .../rocksdb/t/insert_optimized_config-master.opt | 8 - .../rocksdb/t/insert_optimized_config.test | 10 + storage/rocksdb/mysql-test/rocksdb/t/lock.test | 22 + .../t/percona_nonflushing_analyze_debug.test | 11 + .../rocksdb/t/prefix_extractor_override.test | 14 +- storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test | 3 +- .../mysql-test/rocksdb/t/rocksdb_checksums.test | 10 +- .../rocksdb/t/rocksdb_deadlock_detect.inc | 35 +- .../rocksdb/t/secondary_key_update_lock.test | 26 + .../mysql-test/rocksdb/t/set_checkpoint.inc | 2 +- .../rocksdb/mysql-test/rocksdb/t/show_engine.test | 13 + .../rocksdb/mysql-test/rocksdb/t/transaction.test | 23 + .../rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test | 4 +- storage/rocksdb/mysql-test/rocksdb/t/update.test | 10 + .../mysql-test/rocksdb/t/use_direct_reads.test | 37 + .../mysql-test/rocksdb/t/validate_datadic.test | 16 +- .../rocksdb/mysql-test/rocksdb_rpl/combinations | 8 +- .../r/rpl_missing_columns_sk_update.result | 62 + .../rpl_mts_dependency_unique_key_conflicts.result | 44 + .../rocksdb_rpl/r/rpl_rocksdb_snapshot.result | 4 +- .../t/rpl_gtid_crash_safe_wal_corrupt.inc | 2 +- .../t/rpl_missing_columns_sk_update.cnf | 13 + .../t/rpl_missing_columns_sk_update.test | 69 ++ .../t/rpl_mts_dependency_unique_key_conflicts.test | 64 + .../t/rpl_rocksdb_2pc_crash_recover.test | 1 + .../rocksdb/mysql-test/rocksdb_stress/combinations | 5 + .../r/rocksdb_block_cache_size_basic.result | 84 +- .../r/rocksdb_bulk_load_allow_sk_basic.result | 100 ++ ...sdb_commit_time_batch_for_recovery_basic.result | 121 ++ .../r/rocksdb_create_checkpoint_basic.result | 2 +- ...ksdb_debug_manual_compaction_delay_basic.result | 46 + ...ksdb_error_on_suboptimal_collation_basic.result | 7 + .../rocksdb_manual_compaction_threads_basic.result | 93 ++ .../r/rocksdb_max_manifest_file_size_basic.result | 2 +- .../r/rocksdb_max_manual_compactions_basic.result | 57 + .../r/rocksdb_max_row_locks_basic.result | 21 + .../r/rocksdb_stats_recalc_rate_basic.result | 53 + .../r/rocksdb_write_batch_max_bytes_basic.result | 4 +- .../r/rocksdb_write_policy_basic.result | 15 + .../t/rocksdb_block_cache_size_basic.test | 16 +- .../t/rocksdb_bulk_load_allow_sk_basic.test | 18 + ...cksdb_commit_time_batch_for_recovery_basic.test | 20 + .../t/rocksdb_create_checkpoint_basic.test | 2 +- ...ocksdb_debug_manual_compaction_delay_basic.test | 16 + ...ocksdb_error_on_suboptimal_collation_basic.test | 6 + .../t/rocksdb_manual_compaction_threads_basic.test | 17 + .../t/rocksdb_max_manual_compactions_basic.test | 17 + .../t/rocksdb_max_row_locks_basic.test | 1 + .../t/rocksdb_stats_recalc_rate_basic.test | 17 + .../t/rocksdb_write_batch_max_bytes_basic.test | 4 +- .../t/rocksdb_write_policy_basic.test | 17 + storage/rocksdb/properties_collector.cc | 61 +- storage/rocksdb/properties_collector.h | 3 +- storage/rocksdb/rdb_cf_options.cc | 2 - storage/rocksdb/rdb_comparator.h | 46 +- storage/rocksdb/rdb_datadic.cc | 6 +- storage/rocksdb/rdb_datadic.h | 5 +- storage/rocksdb/rdb_i_s.cc | 167 ++- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_index_merge.cc | 5 + storage/rocksdb/rdb_perf_context.cc | 2 - storage/rocksdb/rdb_psi.cc | 17 +- storage/rocksdb/rdb_psi.h | 10 +- storage/rocksdb/rdb_sst_info.cc | 4 +- storage/rocksdb/rdb_threads.h | 31 +- storage/rocksdb/rdb_utils.cc | 23 - storage/rocksdb/rocksdb | 2 +- 132 files changed, 4286 insertions(+), 639 deletions(-) diff --cc storage/rocksdb/CMakeLists.txt index 2a4ea4d,54b0b1f..2689dc7 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@@ -130,55 -53,64 +130,58 @@@ if(HAVE_THREAD_LOCAL ADD_DEFINITIONS(-DROCKSDB_SUPPORT_THREAD_LOCAL) endif() -SET(ROCKSDB_SOURCES - ha_rocksdb.cc ha_rocksdb.h ha_rocksdb_proto.h +INCLUDE(build_rocksdb.cmake) + +ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib + ha_rocksdb_proto.h logger.h rdb_comparator.h - rdb_datadic.cc rdb_datadic.h - rdb_cf_options.cc rdb_cf_options.h - rdb_cf_manager.cc rdb_cf_manager.h - properties_collector.cc properties_collector.h - event_listener.cc event_listener.h - rdb_i_s.cc rdb_i_s.h - rdb_index_merge.cc rdb_index_merge.h - rdb_io_watchdog.cc rdb_io_watchdog.h - rdb_perf_context.cc rdb_perf_context.h - rdb_mutex_wrapper.cc rdb_mutex_wrapper.h - rdb_psi.h rdb_psi.cc - rdb_sst_info.cc rdb_sst_info.h - rdb_utils.cc rdb_utils.h rdb_buff.h - rdb_threads.cc rdb_threads.h - ${ROCKSDB_LIB_SOURCES} + rdb_cf_options.cc + rdb_cf_options.h + event_listener.cc + event_listener.h + rdb_perf_context.cc + rdb_perf_context.h + rdb_buff.h + rdb_mariadb_port.h ) -IF(WITH_FB_TSAN) - SET(PIC_EXT "_pic") -ELSE() - SET(PIC_EXT "") -ENDIF() +ADD_DEPENDENCIES(rocksdb_aux_lib GenError) -SET(rocksdb_static_libs ) -IF (WITH_SNAPPY) - FIND_LIBRARY(SNAPPY_LIBRARY - NAMES libsnappy${PIC_EXT}.a snappy - HINTS ${WITH_SNAPPY}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${SNAPPY_LIBRARY}) - ADD_DEFINITIONS(-DSNAPPY) -ENDIF() +# MARIAROCKS-TODO: how to properly depend on -lrt ? +TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY}) +if (UNIX AND NOT APPLE) + TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt) +endif() -IF (WITH_LZ4) +TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib) + FIND_LIBRARY(LZ4_LIBRARY + NAMES liblz4${PIC_EXT}.a lz4 + HINTS ${WITH_LZ4}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${LZ4_LIBRARY}) - ADD_DEFINITIONS(-DLZ4) -ENDIF() -IF (WITH_BZ2) - FIND_LIBRARY(BZ2_LIBRARY - NAMES libbz2${PIC_EXT}.a bz2 - HINTS ${WITH_BZ2}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${BZ2_LIBRARY}) - ADD_DEFINITIONS(-DBZIP2) +IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + + # MARIAROCKS_NOT_YET: Add -frtti flag when compiling RocksDB files. + # TODO: is this the right way to do this? + # - SQL layer and storage/rocksdb/*.cc are compiled with -fnortti + # - RocksDB files are compiled with "-fnortti ... -frtti" + # - This causes RocksDB headers to be compiled with different settings: + # = with RTTI when compiling RocksDB + # = without RTTI when compiling storage/rocksdb/*.cc + # + # (facebook/mysql-5.6 just compiles everything without -f*rtti, which means + # everything is compiled with -frtti) + # + # (also had to add -frtti above, because something that event_listener.cc + # includes requires it. So, now everything in MariaRocks is compiled with + # -frtti) + set_source_files_properties(event_listener.cc rdb_cf_options.cc rdb_sst_info.cc + PROPERTIES COMPILE_FLAGS -frtti) ENDIF() -IF (WITH_ZSTD) - SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZSTD_LIBRARY}) - ADD_DEFINITIONS(-DZSTD) +CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU) +IF(HAVE_SCHED_GETCPU) + ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1) ENDIF() IF (WITH_TBB) diff --cc storage/rocksdb/ha_rocksdb.cc index adef71e,c7d7cc0..36054ac --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@@ -506,77 -529,20 +545,85 @@@ static my_bool rocksdb_enable_bulk_load static my_bool rocksdb_print_snapshot_conflict_queries = 0; static my_bool rocksdb_large_prefix = 0; static my_bool rocksdb_allow_to_start_after_corruption = 0; +static char* rocksdb_git_hash; + +char *compression_types_val= + const_cast<char*>(get_rocksdb_supported_compression_types()); + static uint64_t rocksdb_write_policy = + rocksdb::TxnDBWritePolicy::WRITE_COMMITTED; + static my_bool rocksdb_error_on_suboptimal_collation = 1; + static uint32_t rocksdb_stats_recalc_rate = 0; + static uint32_t rocksdb_debug_manual_compaction_delay = 0; + static uint32_t rocksdb_max_manual_compactions = 0; std::atomic<uint64_t> rocksdb_row_lock_deadlocks(0); std::atomic<uint64_t> rocksdb_row_lock_wait_timeouts(0); std::atomic<uint64_t> rocksdb_snapshot_conflict_errors(0); std::atomic<uint64_t> rocksdb_wal_group_syncs(0); + std::atomic<uint64_t> rocksdb_manual_compactions_processed(0); + std::atomic<uint64_t> rocksdb_manual_compactions_running(0); + + +/* + Remove directory with files in it. + Used to remove checkpoint created by mariabackup. +*/ +#ifdef _WIN32 +#include <direct.h> /* unlink*/ +#ifndef F_OK +#define F_OK 0 +#endif +#endif + +static int rmdir_force(const char *dir) { + if (access(dir, F_OK)) + return true; + + char path[FN_REFLEN]; + char sep[] = {FN_LIBCHAR, 0}; + int err = 0; + + MY_DIR *dir_info = my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i = 0; i < dir_info->number_of_files; i++) { + FILEINFO *file = dir_info->dir_entry + i; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + err = my_delete(path, 0); + + if (err) { + break; + } + } + + my_dirend(dir_info); + + if (!err) + err = rmdir(dir); + + return (err == 0) ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; +} + + +static void rocksdb_remove_mariabackup_checkpoint( + my_core::THD *const, + struct st_mysql_sys_var *const , + void *const var_ptr, const void *const) { + std::string mariabackup_checkpoint_dir(rocksdb_datadir); + + mariabackup_checkpoint_dir.append("/mariabackup-checkpoint"); + + if (unlink(mariabackup_checkpoint_dir.c_str()) == 0) + return; + + rmdir_force(mariabackup_checkpoint_dir.c_str()); +} + + static std::unique_ptr<rocksdb::DBOptions> rdb_init_rocksdb_db_options(void) { auto o = std::unique_ptr<rocksdb::DBOptions>(new rocksdb::DBOptions()); @@@ -1077,9 -1076,11 +1165,11 @@@ static MYSQL_SYSVAR_INT(table_cache_num "DBOptions::table_cache_numshardbits for RocksDB", nullptr, nullptr, rocksdb_db_options->table_cache_numshardbits, - /* min */ 0, /* max */ INT_MAX, 0); + // LRUCache limits this to 19 bits, anything greater + // fails to create a cache and returns a nullptr + /* min */ 0, /* max */ 19, 0); -static MYSQL_SYSVAR_ULONG(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, +static MYSQL_SYSVAR_UINT64_T(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "DBOptions::WAL_ttl_seconds for RocksDB", nullptr, nullptr, rocksdb_db_options->WAL_ttl_seconds, @@@ -1753,7 -1772,11 +1873,12 @@@ static struct st_mysql_sys_var *rocksdb MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(allow_to_start_after_corruption), + MYSQL_SYSVAR(git_hash), + MYSQL_SYSVAR(error_on_suboptimal_collation), + MYSQL_SYSVAR(stats_recalc_rate), + MYSQL_SYSVAR(debug_manual_compaction_delay), + MYSQL_SYSVAR(max_manual_compactions), + MYSQL_SYSVAR(manual_compaction_threads), nullptr}; static rocksdb::WriteOptions @@@ -1897,9 -1941,7 +2064,9 @@@ private ulonglong m_writes_at_last_savepoint; protected: - THD *m_thd = nullptr; + - rocksdb::ReadOptions m_read_opts; ++protected: + THD *m_thd = nullptr; static std::multiset<Rdb_transaction *> s_tx_list; static mysql_mutex_t s_tx_list_mutex; @@@ -1956,14 -1990,20 +2123,23 @@@ protected return s; } + protected: + /* + The following two are helper functions to be overloaded by child classes. + They should provide RocksDB's savepoint semantics. + */ + virtual void do_set_savepoint() = 0; + virtual void do_rollback_to_savepoint() = 0; + public: + rocksdb::ReadOptions m_read_opts; const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; +#ifdef MARIAROCKS_NOT_YET + // TODO: MariaDB probably doesn't need these at all: const char *m_mysql_gtid; const char *m_mysql_max_gtid; +#endif String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; @@@ -2946,6 -2991,6 +3141,7 @@@ error m_batch->RollbackToSavePoint(); } ++ public: bool is_writebatch_trx() const override { return true; } @@@ -3563,7 -3494,8 +3766,8 @@@ static int rocksdb_commit(handlerton* h /* We get here when committing a statement within a transaction. */ - tx->set_tx_failed(false); + tx->make_stmt_savepoint_permanent(); + tx->make_stmt_savepoint_permanent(); } if (my_core::thd_tx_isolation(thd) <= ISO_READ_COMMITTED) { @@@ -4119,9 -4052,24 +4326,25 @@@ static bool rocksdb_show_status(handler str, stat_print); } } - #endif + /* Explicit snapshot information */ + str.clear(); + { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + for (const auto &elem : explicit_snapshots) { + const auto &ss = elem.second.lock(); + DBUG_ASSERT(ss != nullptr); + const auto &info = ss->ss_info; + str += "\nSnapshot ID: " + std::to_string(info.snapshot_id) + + "\nBinlog File: " + info.binlog_file + + "\nBinlog Pos: " + std::to_string(info.binlog_pos) + + "\nGtid Executed: " + info.gtid_executed + "\n"; + } + } + if (!str.empty()) { + res |= print_stats(thd, "EXPLICIT_SNAPSHOTS", "rocksdb", str, stat_print); + } +#ifdef MARIAROCKS_NOT_YET } else if (stat_type == HA_ENGINE_TRX) { /* Handle the SHOW ENGINE ROCKSDB TRANSACTION STATUS command */ res |= rocksdb_show_snapshot_status(hton, thd, stat_print); @@@ -4141,8 -4089,48 +4364,50 @@@ static inline void rocksdb_register_tx( } } +static const char *ha_rocksdb_exts[] = {NullS}; + + static bool rocksdb_explicit_snapshot( + handlerton *const /* hton */, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle */ + snapshot_info_st *ss_info) /*!< out: Snapshot information */ + { + switch (ss_info->op) { + case snapshot_operation::SNAPSHOT_CREATE: { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + auto s = Rdb_explicit_snapshot::create(ss_info, rdb, rdb->GetSnapshot()); + if (mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + thd->set_explicit_snapshot(s); + return s == nullptr; + } + case snapshot_operation::SNAPSHOT_ATTACH: { + auto s = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!s) { + return true; + } + *ss_info = s->ss_info; + thd->set_explicit_snapshot(s); + return false; + } + case snapshot_operation::SNAPSHOT_RELEASE: { + if (!thd->get_explicit_snapshot()) { + return true; + } + *ss_info = thd->get_explicit_snapshot()->ss_info; + thd->set_explicit_snapshot(nullptr); + return false; + } + default: + DBUG_ASSERT(false); + return true; + } + return true; + } + /* Supporting START TRANSACTION WITH CONSISTENT [ROCKSDB] SNAPSHOT @@@ -4176,16 -4166,14 +4443,27 @@@ static int rocksdb_start_tx_and_assign_ my_error(ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT, MYF(0)); return HA_EXIT_FAILURE; } + ++#ifdef MARIADB_NOT_YET + if (ss_info) { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } else { + return HA_EXIT_FAILURE; + } - } ++#endif ++ + /* + MariaDB: there is no need to call mysql_bin_log_lock_commits and then + unlock back. + SQL layer calls start_consistent_snapshot() for all engines, including the + binlog under LOCK_commit_ordered mutex. ++ + The mutex prevents binlog commits from happening (right?) while the storage + engine(s) allocate read snapshots. That way, each storage engine is - synchronized with current binlog position. ++ synchronized with current binlog position. + */ + mysql_mutex_assert_owner(&LOCK_commit_ordered); Rdb_transaction *const tx = get_or_create_tx(thd); Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); @@@ -4195,6 -4183,10 +4473,11 @@@ rocksdb_register_tx(hton, thd, tx); tx->acquire_snapshot(true); ++#ifdef MARIADB_NOT_YET + if (ss_info) { + mysql_bin_log_unlock_commits(ss_info); + } - ++#endif return HA_EXIT_SUCCESS; } @@@ -4445,6 -4502,9 +4811,8 @@@ static int rocksdb_init_func(void *cons mysql_mutex_init(rdb_sysvars_psi_mutex_key, &rdb_sysvars_mutex, MY_MUTEX_INIT_FAST); + mysql_mutex_init(rdb_block_cache_resize_mutex_key, + &rdb_block_cache_resize_mutex, MY_MUTEX_INIT_FAST); - rdb_open_tables.init_hash(); Rdb_transaction::init_mutex(); rocksdb_hton->state = SHOW_OPTION_YES; @@@ -4457,16 -4514,14 +4825,18 @@@ rocksdb_hton->commit_by_xid = rocksdb_commit_by_xid; rocksdb_hton->rollback_by_xid = rocksdb_rollback_by_xid; rocksdb_hton->recover = rocksdb_recover; + + rocksdb_hton->commit_ordered= rocksdb_commit_ordered; rocksdb_hton->commit = rocksdb_commit; + + rocksdb_hton->commit_checkpoint_request= rocksdb_checkpoint_request; + rocksdb_hton->rollback = rocksdb_rollback; - rocksdb_hton->db_type = DB_TYPE_ROCKSDB; rocksdb_hton->show_status = rocksdb_show_status; + rocksdb_hton->explicit_snapshot = rocksdb_explicit_snapshot; rocksdb_hton->start_consistent_snapshot = rocksdb_start_tx_and_assign_read_view; + rocksdb_hton->start_shared_snapshot = rocksdb_start_tx_with_shared_read_view; rocksdb_hton->savepoint_set = rocksdb_savepoint; rocksdb_hton->savepoint_rollback = rocksdb_rollback_to_savepoint; rocksdb_hton->savepoint_rollback_can_release_mdl = @@@ -4710,9 -4791,10 +5111,9 @@@ } cf_manager.init(std::move(cf_options_map), &cf_handles); - if (dict_manager.init(rdb->GetBaseDB(), &cf_manager)) { + if (dict_manager.init(rdb, &cf_manager)) { // NO_LINT_DEBUG sql_print_error("RocksDB: Failed to initialize data dictionary."); - rdb_open_tables.free_hash(); DBUG_RETURN(HA_EXIT_FAILURE); } @@@ -4877,66 -4968,28 +5293,77 @@@ static int rocksdb_done_func(void *cons sql_print_error("RocksDB: Couldn't stop the index thread: (errno=%d)", err); } + // signal the manual compaction thread to stop + rdb_mc_thread.signal(true); + // Wait for the manual compaction thread to finish. + err = rdb_mc_thread.join(); + if (err != 0) { + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Couldn't stop the manual compaction thread: (errno=%d)", err); + } + - if (rdb_open_tables.m_hash.records) { + if (rdb_open_tables.m_hash.size()) { // Looks like we are getting unloaded and yet we have some open tables // left behind. error = 1; } - rdb_open_tables.free_hash(); + /* + MariaDB: When the plugin is unloaded with UNINSTALL SONAME command, some + connections may still have Rdb_transaction objects. + + These objects are not genuine transactions (as SQL layer makes sure that + a plugin that is being unloaded has no open tables), they are empty + Rdb_transaction objects that were left there to save on object + creation/deletion. + + Go through the list and delete them. + */ + { + class Rdb_trx_deleter: public Rdb_tx_list_walker { + public: + std::set<Rdb_transaction*> rdb_trxs; + + void process_tran(const Rdb_transaction *const tx) override { + /* + Check if the transaction is really empty. We only check + non-WriteBatch-based transactions, because there is no easy way to + check WriteBatch-based transactions. + */ + if (!tx->is_writebatch_trx()) { + const auto tx_impl = static_cast<const Rdb_transaction_impl *>(tx); + DBUG_ASSERT(tx_impl); + if (tx_impl->get_rdb_trx()) + DBUG_ASSERT(0); + } + rdb_trxs.insert((Rdb_transaction*)tx); + }; + } deleter; + + Rdb_transaction::walk_tx_list(&deleter); + + for (std::set<Rdb_transaction*>::iterator it= deleter.rdb_trxs.begin(); + it != deleter.rdb_trxs.end(); + ++it) + { + // When a transaction is deleted, it removes itself from s_tx_list. + delete *it; + } + } + + /* + destructors for static objects can be called at _exit(), + but we want to free the memory at dlclose() + */ + rdb_open_tables.m_hash.~Rdb_table_set(); mysql_mutex_destroy(&rdb_open_tables.m_mutex); mysql_mutex_destroy(&rdb_sysvars_mutex); + mysql_mutex_destroy(&rdb_block_cache_resize_mutex); + delete rdb_collation_exceptions; + mysql_mutex_destroy(&rdb_collation_data_mutex); mysql_mutex_destroy(&rdb_mem_cmp_space_mutex); @@@ -6307,14 -6348,13 +6729,14 @@@ void ha_rocksdb::free_key_buffers() my_free(m_dup_sk_packed_tuple_old); m_dup_sk_packed_tuple_old = nullptr; - my_free(m_eq_cond_upper_bound); - m_eq_cond_upper_bound = nullptr; + my_free(m_scan_it_lower_bound); + m_scan_it_lower_bound = nullptr; - my_free(m_eq_cond_lower_bound); - m_eq_cond_lower_bound = nullptr; + my_free(m_scan_it_upper_bound); + m_scan_it_upper_bound = nullptr; } +#ifdef MARIAROCKS_NOT_YET void ha_rocksdb::set_use_read_free_rpl(const char *const whitelist) { DBUG_ASSERT(whitelist != nullptr); @@@ -6796,12 -6766,12 +7216,10 @@@ int ha_rocksdb::create_cfs std::array<struct key_def_cf_info, MAX_INDEXES + 1> *const cfs) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); char tablename_sys[NAME_LEN + 1]; - - my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), - tablename_sys, sizeof(tablename_sys)); + bool tsys_set= false; /* The first loop checks the index parameters and creates @@@ -6810,40 -6780,37 +7228,49 @@@ for (uint i = 0; i < tbl_def_arg->m_key_count; i++) { rocksdb::ColumnFamilyHandle *cf_handle; - if (rocksdb_strict_collation_check && - !is_hidden_pk(i, table_arg, tbl_def_arg) && + if (!is_hidden_pk(i, table_arg, tbl_def_arg) && tbl_def_arg->base_tablename().find(tmp_file_prefix) != 0) { - for (uint part = 0; part < table_arg->key_info[i].actual_key_parts; - part++) { - if (!rdb_is_index_collation_supported( + if (!tsys_set) + { + tsys_set= true; + my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), + tablename_sys, sizeof(tablename_sys)); + } + + for (uint part = 0; part < table_arg->key_info[i].ext_key_parts; + part++) + { + /* MariaDB: disallow NOPAD collations */ + if (rdb_field_uses_nopad_collation( + table_arg->key_info[i].key_part[part].field)) + { + my_error(ER_MYROCKS_CANT_NOPAD_COLLATION, MYF(0)); + DBUG_RETURN(HA_EXIT_FAILURE); + } + + if (rocksdb_strict_collation_check && + !rdb_is_index_collation_supported( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - std::string collation_err; - for (const auto &coll : RDB_INDEX_COLLATIONS) { - if (collation_err != "") { - collation_err += ", "; - } - collation_err += coll->name; - } - if (rocksdb_error_on_suboptimal_collation) { - my_error(ER_UNSUPPORTED_COLLATION, MYF(0), - tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name, - collation_err.c_str()); - DBUG_RETURN(HA_EXIT_FAILURE); + char buf[1024]; + my_snprintf(buf, sizeof(buf), + "Indexed column %s.%s uses a collation that does not " + "allow index-only access in secondary key and has " + "reduced disk space efficiency in primary key.", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name); + + my_error(ER_INTERNAL_ERROR, MYF(ME_JUST_WARNING), buf); + } else { + push_warning_printf( + ha_thd(), Sql_condition::WARN_LEVEL_WARN, ER_WRONG_ARGUMENTS, + "Unsupported collation on string indexed column %s.%s Use " + "binary collation (%s).", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name, + collation_err.c_str()); + } } } } @@@ -7212,10 -7163,7 +7623,8 @@@ int ha_rocksdb::create_key_def(const TA int rdb_normalize_tablename(const std::string &tablename, std::string *const strbuf) { - DBUG_ASSERT(strbuf != nullptr); - - if (tablename.size() < 2 || tablename[0] != '.' || tablename[1] != '/') { + if (tablename.size() < 2 || tablename[0] != '.' || + (tablename[1] != FN_LIBCHAR && tablename[1] != FN_LIBCHAR2)) { DBUG_ASSERT(0); // We were not passed table name? return HA_ERR_ROCKSDB_INVALID_TABLE; } @@@ -7875,11 -7809,10 +8278,10 @@@ ulong ha_rocksdb::index_flags(uint inx pair for. */ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { - DBUG_ASSERT(buf != nullptr); DBUG_ASSERT(table != nullptr); - +#ifdef MARIAROCKS_NOT_YET stats.rows_requested++; - +#endif /* Use STATUS_NOT_FOUND when record not found or some error occurred */ table->status = STATUS_NOT_FOUND; @@@ -9798,34 -9666,23 +10187,28 @@@ int ha_rocksdb::update_write_row(const If the index was reverse order, upper bound would be 0x0000b3eb003f65c5e78857, and lower bound would be 0x0000b3eb003f65c5e78859. These cover given eq condition range. + + @param lower_bound_buf IN Buffer for lower bound + @param upper_bound_buf IN Buffer for upper bound + + @param outer_u */ - void ha_rocksdb::setup_iterator_bounds(const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { - uint eq_cond_len = eq_cond.size(); - memcpy(upper_bound_buf, eq_cond.data(), eq_cond_len); - kd.successor(upper_bound_buf, eq_cond_len); - memcpy(lower_bound_buf, eq_cond.data(), eq_cond_len); - kd.predecessor(lower_bound_buf, eq_cond_len); + void ha_rocksdb::setup_iterator_bounds( + const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, size_t bound_len, + uchar *const lower_bound, uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, rocksdb::Slice *upper_bound_slice) { + uint min_len = std::min(eq_cond.size(), bound_len); + memcpy(upper_bound, eq_cond.data(), min_len); + kd.successor(upper_bound, min_len); + memcpy(lower_bound, eq_cond.data(), min_len); + kd.predecessor(lower_bound, min_len); if (kd.m_is_reverse_cf) { - *out_upper_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); } else { - *out_upper_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); } } @@@ -9845,11 -9701,10 +10227,11 @@@ void ha_rocksdb::setup_scan_iterator(co bool skip_bloom = true; const rocksdb::Slice eq_cond(slice->data(), eq_cond_len); - if (check_bloom_and_set_bounds(ha_thd(), kd, eq_cond, use_all_keys, - m_eq_cond_lower_bound, - m_eq_cond_upper_bound, - &m_eq_cond_lower_bound_slice, + if (check_bloom_and_set_bounds( + ha_thd(), kd, eq_cond, use_all_keys, eq_cond_len, + m_scan_it_lower_bound, m_scan_it_upper_bound, + &m_scan_it_lower_bound_slice, &m_scan_it_upper_bound_slice)) { + &m_eq_cond_upper_bound_slice)) { skip_bloom = false; } @@@ -9955,9 -9810,10 +10337,9 @@@ int ha_rocksdb::rnd_next(uchar *const b DBUG_ENTER_FUNC(); int rc; - ha_statistic_increment(&SSV::ha_read_rnd_next_count); for (;;) { rc = rnd_next_with_direction(buf, true); - if (rc != HA_ERR_LOCK_DEADLOCK || !m_rnd_scan_is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !m_rnd_scan_is_new_snapshot) break; /* exit the loop */ // release the snapshot and iterator and then regenerate them @@@ -10362,12 -10214,23 +10744,23 @@@ int ha_rocksdb::info(uint flag) continue; } KEY *const k = &table->key_info[i]; - for (uint j = 0; j < k->actual_key_parts; j++) { + for (uint j = 0; j < k->ext_key_parts; j++) { const Rdb_index_stats &k_stats = m_key_descr_arr[i]->m_stats; - uint x = k_stats.m_distinct_keys_per_prefix.size() > j && - k_stats.m_distinct_keys_per_prefix[j] > 0 - ? k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j] - : 0; + uint x; + + if (k_stats.m_distinct_keys_per_prefix.size() > j && + k_stats.m_distinct_keys_per_prefix[j] > 0) { + x = k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j]; + /* + If the number of rows is less than the number of prefixes (due to + sampling), the average number of rows with the same prefix is 1. + */ + if (x == 0) { + x = 1; + } + } else { + x = 0; + } if (x > stats.records) x = stats.records; if ((x == 0 && rocksdb_debug_optimizer_no_zero_cardinality) || @@@ -12807,24 -12661,198 +13225,195 @@@ void Rdb_background_thread::run() rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); } } - + // Recalculate statistics for indexes. + if (rocksdb_stats_recalc_rate) { + std::unordered_map<GL_INDEX_ID, std::shared_ptr<const Rdb_key_def>> + to_recalc; + + if (rdb_indexes_to_recalc.empty()) { + struct Rdb_index_collector : public Rdb_tables_scanner { + int add_table(Rdb_tbl_def *tdef) override { + for (uint i = 0; i < tdef->m_key_count; i++) { + rdb_indexes_to_recalc.push_back( + tdef->m_key_descr_arr[i]->get_gl_index_id()); + } + return HA_EXIT_SUCCESS; + } + } collector; + ddl_manager.scan_for_tables(&collector); + } + + while (to_recalc.size() < rocksdb_stats_recalc_rate && + !rdb_indexes_to_recalc.empty()) { + const auto index_id = rdb_indexes_to_recalc.back(); + rdb_indexes_to_recalc.pop_back(); + + std::shared_ptr<const Rdb_key_def> keydef = + ddl_manager.safe_find(index_id); + + if (keydef) { + to_recalc.insert(std::make_pair(keydef->get_gl_index_id(), keydef)); + } + } + + if (!to_recalc.empty()) { + calculate_stats(to_recalc, false); + } + } + - // Set the next timestamp for mysql_cond_timedwait() (which ends up calling - // pthread_cond_timedwait()) to wait on. - ts_next_sync.tv_sec = ts.tv_sec + WAKE_UP_INTERVAL; } // save remaining stats which might've left unsaved ddl_manager.persist_stats(); } - bool ha_rocksdb::check_bloom_and_set_bounds(THD *thd, const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - const bool use_all_keys, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { + /* + A background thread to handle manual compactions, + except for dropping indexes/tables. Every second, it checks + pending manual compactions, and it calls CompactRange if there is. + */ + void Rdb_manual_compaction_thread::run() { + mysql_mutex_init(0, &m_mc_mutex, MY_MUTEX_INIT_FAST); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + for (;;) { + if (m_stop) { + break; + } + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 1; + + const auto ret MY_ATTRIBUTE((__unused__)) = + mysql_cond_timedwait(&m_signal_cond, &m_signal_mutex, &ts); + if (m_stop) { + break; + } + // make sure, no program error is returned + DBUG_ASSERT(ret == 0 || ret == ETIMEDOUT); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + // Grab the first item and proceed, if not empty. + if (m_requests.empty()) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + continue; + } + Manual_compaction_request &mcr = m_requests.begin()->second; + DBUG_ASSERT(mcr.cf != nullptr); + DBUG_ASSERT(mcr.state == Manual_compaction_request::INITED); + mcr.state = Manual_compaction_request::RUNNING; + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + + DBUG_ASSERT(mcr.state == Manual_compaction_request::RUNNING); + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s started.", mcr.mc_id, + mcr.cf->GetName().c_str()); + rocksdb_manual_compactions_running++; + if (rocksdb_debug_manual_compaction_delay > 0) { + my_sleep(rocksdb_debug_manual_compaction_delay * 1000000); + } + // CompactRange may take a very long time. On clean shutdown, + // it is cancelled by CancelAllBackgroundWork, then status is + // set to shutdownInProgress. + const rocksdb::Status s = rdb->CompactRange( + getCompactRangeOptions(mcr.concurrency), mcr.cf, mcr.start, mcr.limit); + rocksdb_manual_compactions_running--; + if (s.ok()) { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s ended.", mcr.mc_id, + mcr.cf->GetName().c_str()); + } else { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s aborted. %s", + mcr.mc_id, mcr.cf->GetName().c_str(), s.getState()); + if (!s.IsShutdownInProgress()) { + rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + } else { + DBUG_ASSERT(m_requests.size() == 1); + } + } + rocksdb_manual_compactions_processed++; + clear_manual_compaction_request(mcr.mc_id, false); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + } + clear_all_manual_compaction_requests(); + DBUG_ASSERT(m_requests.empty()); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + mysql_mutex_destroy(&m_mc_mutex); + } + + void Rdb_manual_compaction_thread::clear_all_manual_compaction_requests() { + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + m_requests.clear(); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + } + + void Rdb_manual_compaction_thread::clear_manual_compaction_request( + int mc_id, bool init_only) { + bool erase = true; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + if (init_only) { + Manual_compaction_request mcr = it->second; + if (mcr.state != Manual_compaction_request::INITED) { + erase = false; + } + } + if (erase) { + m_requests.erase(it); + } + } else { + // Current code path guarantees that erasing by the same mc_id happens + // at most once. INITED state may be erased by a thread that requested + // the compaction. RUNNING state is erased by mc thread only. + DBUG_ASSERT(0); + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + } + + int Rdb_manual_compaction_thread::request_manual_compaction( + rocksdb::ColumnFamilyHandle *cf, rocksdb::Slice *start, + rocksdb::Slice *limit, int concurrency) { + int mc_id = -1; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.size() >= rocksdb_max_manual_compactions) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + Manual_compaction_request mcr; + mc_id = mcr.mc_id = ++m_latest_mc_id; + mcr.state = Manual_compaction_request::INITED; + mcr.cf = cf; + mcr.start = start; + mcr.limit = limit; + mcr.concurrency = concurrency; + m_requests.insert(std::make_pair(mcr.mc_id, mcr)); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + + bool Rdb_manual_compaction_thread::is_manual_compaction_finished(int mc_id) { + bool finished = false; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.count(mc_id) == 0) { + finished = true; + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return finished; + } + + bool ha_rocksdb::check_bloom_and_set_bounds( + THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, + const bool use_all_keys, size_t bound_len, uchar *const lower_bound, + uchar *const upper_bound, rocksdb::Slice *lower_bound_slice, + rocksdb::Slice *upper_bound_slice) { bool can_use_bloom = can_use_bloom_filter(thd, kd, eq_cond, use_all_keys); if (!can_use_bloom) { - setup_iterator_bounds(kd, eq_cond, - lower_bound_buf, upper_bound_buf, + setup_iterator_bounds(kd, eq_cond, bound_len, lower_bound, upper_bound, + lower_bound_slice, upper_bound_slice); + out_lower_bound, out_upper_bound); } return can_use_bloom; } @@@ -12973,11 -13000,6 +13561,8 @@@ const char *get_rdb_io_error_string(con // In case of core dump generation we want this function NOT to be optimized // so that we can capture as much data as possible to debug the root cause // more efficiently. +#ifdef __GNUC__ - #pragma GCC push_options - #pragma GCC optimize("O0") +#endif - void rdb_handle_io_error(const rocksdb::Status status, const RDB_IO_ERROR_TYPE err_type) { if (status.IsIOError()) { @@@ -13026,10 -13051,7 +13614,8 @@@ } } } - +#ifdef __GNUC__ - #pragma GCC pop_options +#endif - Rdb_dict_manager *rdb_get_dict_manager(void) { return &dict_manager; } Rdb_ddl_manager *rdb_get_ddl_manager(void) { return &ddl_manager; } @@@ -13573,7 -13605,6 +14202,7 @@@ maria_declare_plugin(rocksdb_se) myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, myrocks::rdb_i_s_cfoptions, myrocks::rdb_i_s_compact_stats, myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, - myrocks::rdb_i_s_index_file_map, myrocks::rdb_i_s_lock_info, - myrocks::rdb_i_s_trx_info, + myrocks::rdb_i_s_sst_props, myrocks::rdb_i_s_index_file_map, + myrocks::rdb_i_s_lock_info, myrocks::rdb_i_s_trx_info, - myrocks::rdb_i_s_deadlock_info mysql_declare_plugin_end; + myrocks::rdb_i_s_deadlock_info +maria_declare_plugin_end; diff --cc storage/rocksdb/ha_rocksdb.h index 739a2c8,ca14165..c444710 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@@ -1402,8 -1369,7 +1407,8 @@@ public virtual void rpl_after_delete_rows() override; virtual void rpl_before_update_rows() override; virtual void rpl_after_update_rows() override; - virtual bool use_read_free_rpl(); + virtual bool use_read_free_rpl() override; +#endif // MARIAROCKS_NOT_YET private: /* Flags tracking if we are inside different replication operation */ diff --cc storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result index 08f2329,9731007..0617232 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result @@@ -6,18 -6,16 +6,18 @@@ LOAD DATA INFILE <input_file> INTO TABL set rocksdb_bulk_load=0; select count(pk) from t1; count(pk) -3000000 +300000 select count(a) from t1; count(a) -3000000 +300000 select count(b) from t1; count(b) -3000000 +300000 ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; - ERROR HY000: Status error 10 received from RocksDB: Operation aborted: Failed to acquire lock due to max_num_locks limit + ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to max_num_locks limit' from ROCKSDB set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=0; diff --cc storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result index a8d5c07,6f811e1..de8a1fb --- a/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result @@@ -276,16 -276,12 +276,16 @@@ 10 20 30 - connect con1,localhost,root,,; - connection con1; alter table t1 force; - connection default; + select * from t1; + a + insert into t1 values (100); select * from t1; a +connection con1; +insert into t1 values (100); +select * from t1; +a 1 2 100 diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result index fe08cd7,8b51c12..604e557 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result @@@ -56,52 -56,44 +56,44 @@@ insert into t values () begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_before"; +set debug_dbug="+d,crash_commit_before"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After engine prepare begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_prepare"; +set debug_dbug="+d,crash_commit_after_prepare"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After binlog begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_log"; +set debug_dbug="+d,crash_commit_after_log"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 6 - select max(i) from t; - max(i) - 5 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After everything begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after"; +set debug_dbug="+d,crash_commit_after"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 8 - select max(i) from t; - max(i) - 7 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 0c49622,17e6bed..f59b841 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@@ -150,11 -151,11 +150,29 @@@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +#--------------------------------------------------------------- +# MDEV-16703 Assertion failed in load_auto_incr_value_from_index +#--------------------------------------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++#---------------------------------- ++# Issue #792 Crash in autoincrement ++#---------------------------------- ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; + #---------------------------------- + # Issue #869 Crash in autoincrement + #---------------------------------- + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/collation.result index e372cbe,b4354ad..10e0d9b --- a/storage/rocksdb/mysql-test/rocksdb/r/collation.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/collation.result @@@ -1,11 -1,12 +1,12 @@@ - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - DROP TABLE IF EXISTS t1; + call mtr.add_suppression("Invalid pattern"); CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). DROP TABLE t1; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (latin1_bin, binary, utf8_bin). CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (latin1_bin, binary, utf8_bin). SET GLOBAL rocksdb_strict_collation_check=0; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; DROP TABLE t1; @@@ -124,7 -125,19 +126,19 @@@ ERROR HY000: Unsupported collation on s SET GLOBAL rocksdb_strict_collation_exceptions="abc"; CREATE TABLE abc (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; CREATE TABLE abcd (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (latin1_bin, binary, utf8_bin). DROP TABLE abc; SET GLOBAL rocksdb_strict_collation_exceptions=null; - SET GLOBAL rocksdb_strict_collation_exceptions=@start_global_value; + SET GLOBAL rocksdb_strict_collation_check=1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset latin1 collate latin1_bin; + ALTER TABLE t1 collate=latin1_general_ci; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result index 36db920,2a00fcb..0619e2d --- a/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result @@@ -13,8 -9,9 +13,9 @@@ connection default show create table information_schema.rocksdb_deadlock; Table Create Table ROCKSDB_DEADLOCK CREATE TEMPORARY TABLE `ROCKSDB_DEADLOCK` ( - `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT '0', + `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT 0, + `TIMESTAMP` bigint(8) NOT NULL DEFAULT '0', - `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT '0', + `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT 0, `CF_NAME` varchar(193) NOT NULL DEFAULT '', `WAITING_KEY` varchar(513) NOT NULL DEFAULT '', `LOCK_TYPE` varchar(193) NOT NULL DEFAULT '', @@@ -25,9 -22,8 +26,9 @@@ create table t (i int primary key) engine=rocksdb; insert into t values (1), (2), (3); select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #1 +connection con1; begin; select * from t where i=1 for update; i @@@ -47,13 -39,11 +48,13 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 Deadlock #2 +connection con1; begin; select * from t where i=1 for update; i @@@ -73,16 -59,14 +74,16 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 10; Deadlock #3 +connection con1; begin; select * from t where i=1 for update; i @@@ -102,24 -82,21 +103,24 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 1; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 +connection con3; set rocksdb_deadlock_detect_depth = 2; Deadlock #4 +connection con1; begin; select * from t where i=1 for update; i @@@ -150,12 -120,10 +151,12 @@@ connection con1 i 2 rollback; +connection default; set global rocksdb_max_latest_deadlocks = 5; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #5 +connection con1; begin; select * from t where i=1 for update; i @@@ -191,16 -154,11 +192,16 @@@ connection con1 i 2 rollback; +connection con3; rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 +disconnect con1; +disconnect con2; +disconnect con3; set global rocksdb_lock_wait_timeout = @prior_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_deadlock_detect; drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result index 4370715,80d95e4..e024873 --- a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result @@@ -266,8 -266,8 +266,8 @@@ explai select * from t0,t1 where (t0.key1=t1.key1) and (t0.key1=3 or t0.key2<4) and t1.key1=2; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t0 ref i1,i2 i1 4 const 1 Using where + 1 SIMPLE t0 ref i1,i2 i1 4 const 2 Using where -1 SIMPLE t1 ref i1 i1 4 const 2 NULL +1 SIMPLE t1 ref i1 i1 4 const 1 explain select * from t0,t1 where t0.key1 = 5 and (t1.key1 = t0.key1 or t1.key8 = t0.key1); id select_type table type possible_keys key key_len ref rows Extra @@@ -281,12 -281,13 +281,12 @@@ id select_type table type possible_key explain select * from t1 where key1=3 or key2=4 union select * from t1 where key1<4 or key3=5; id select_type table type possible_keys key key_len ref rows Extra - 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where - 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 2 Using sort_union(i1,i3); Using where + 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where + 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 4 Using sort_union(i1,i3); Using where -NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL Using temporary +NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL explain select * from (select * from t1 where key1 = 3 or key2 =3) as Z where key8 >5; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY <derived2> ALL NULL NULL NULL NULL 4 Using where -2 DERIVED t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where +1 SIMPLE t1 index_merge i1,i2,i8 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where create table t3 like t0; insert into t3 select * from t0; alter table t3 add key9 int not null, add index i9(key9); @@@ -1321,16 -1324,16 +1321,16 @@@ id select_type table type possible_key 1 SIMPLE t1 ref key1 key1 4 const ROWS Using where explain select * from t1 where pk1 < 7500 and key1 = 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 PRIMARY 4 NULL ROWS Using where +1 SIMPLE t1 range PRIMARY,key1 key1 8 NULL ROWS Using index condition explain select * from t1 where pktail1ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail1ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail1ok key1,pktail1ok 4,4 NULL 1 Using intersect(key1,pktail1ok); Using where explain select * from t1 where pktail2ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail2ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail2ok key1,pktail2ok 4,4 NULL 1 Using intersect(key1,pktail2ok); Using where explain select * from t1 where (pktail2ok=1 and pk1< 50000) or key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok pktail2ok,key1 8,4 NULL ROWS Using sort_union(pktail2ok,key1); Using where + 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok PRIMARY,key1 4,4 NULL ROWS Using union(PRIMARY,key1); Using where explain select * from t1 where pktail3bad=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref key1,pktail3bad EITHER_KEY 4 const ROWS Using where diff --cc storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index aba14e3,f830440..e22a85f --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@@ -16,8 -12,12 +16,9 @@@ SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WOR select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index dc13bb3,f9a3f81..c6e15dc --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@@ -915,7 -904,8 +918,8 @@@ rocksdb_enable_ttl O rocksdb_enable_ttl_read_filtering ON rocksdb_enable_write_thread_adaptive_yield OFF rocksdb_error_if_exists OFF + rocksdb_error_on_suboptimal_collation ON -rocksdb_flush_log_at_trx_commit 1 +rocksdb_flush_log_at_trx_commit 0 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF @@@ -1457,111 -1446,117 +1465,115 @@@ drop table t0, t1 # show status like 'rocksdb%'; Variable_name Value -rocksdb_rows_deleted # -rocksdb_rows_inserted # -rocksdb_rows_read # -rocksdb_rows_updated # -rocksdb_rows_deleted_blind # -rocksdb_rows_expired # -rocksdb_rows_filtered # -rocksdb_system_rows_deleted # -rocksdb_system_rows_inserted # -rocksdb_system_rows_read # -rocksdb_system_rows_updated # -rocksdb_memtable_total # -rocksdb_memtable_unflushed # -rocksdb_queries_point # -rocksdb_queries_range # -rocksdb_covered_secondary_key_lookups # -rocksdb_block_cache_add # -rocksdb_block_cache_add_failures # -rocksdb_block_cache_bytes_read # -rocksdb_block_cache_bytes_write # -rocksdb_block_cache_data_add # -rocksdb_block_cache_data_bytes_insert # -rocksdb_block_cache_data_hit # -rocksdb_block_cache_data_miss # -rocksdb_block_cache_filter_add # -rocksdb_block_cache_filter_bytes_evict # -rocksdb_block_cache_filter_bytes_insert # -rocksdb_block_cache_filter_hit # -rocksdb_block_cache_filter_miss # -rocksdb_block_cache_hit # -rocksdb_block_cache_index_add # -rocksdb_block_cache_index_bytes_evict # -rocksdb_block_cache_index_bytes_insert # -rocksdb_block_cache_index_hit # -rocksdb_block_cache_index_miss # -rocksdb_block_cache_miss # -rocksdb_block_cachecompressed_hit # -rocksdb_block_cachecompressed_miss # +Rocksdb_rows_deleted # +Rocksdb_rows_inserted # +Rocksdb_rows_read # +Rocksdb_rows_updated # +Rocksdb_rows_deleted_blind # +Rocksdb_rows_expired # +Rocksdb_rows_filtered # +Rocksdb_system_rows_deleted # +Rocksdb_system_rows_inserted # +Rocksdb_system_rows_read # +Rocksdb_system_rows_updated # +Rocksdb_memtable_total # +Rocksdb_memtable_unflushed # +Rocksdb_queries_point # +Rocksdb_queries_range # +Rocksdb_covered_secondary_key_lookups # +Rocksdb_block_cache_add # +Rocksdb_block_cache_add_failures # +Rocksdb_block_cache_bytes_read # +Rocksdb_block_cache_bytes_write # +Rocksdb_block_cache_data_add # +Rocksdb_block_cache_data_bytes_insert # +Rocksdb_block_cache_data_hit # +Rocksdb_block_cache_data_miss # +Rocksdb_block_cache_filter_add # +Rocksdb_block_cache_filter_bytes_evict # +Rocksdb_block_cache_filter_bytes_insert # +Rocksdb_block_cache_filter_hit # +Rocksdb_block_cache_filter_miss # +Rocksdb_block_cache_hit # +Rocksdb_block_cache_index_add # +Rocksdb_block_cache_index_bytes_evict # +Rocksdb_block_cache_index_bytes_insert # +Rocksdb_block_cache_index_hit # +Rocksdb_block_cache_index_miss # +Rocksdb_block_cache_miss # +Rocksdb_block_cachecompressed_hit # +Rocksdb_block_cachecompressed_miss # + rocksdb_bloom_filter_full_positive # + rocksdb_bloom_filter_full_true_positive # -rocksdb_bloom_filter_prefix_checked # -rocksdb_bloom_filter_prefix_useful # -rocksdb_bloom_filter_useful # -rocksdb_bytes_read # -rocksdb_bytes_written # -rocksdb_compact_read_bytes # -rocksdb_compact_write_bytes # -rocksdb_compaction_key_drop_new # -rocksdb_compaction_key_drop_obsolete # -rocksdb_compaction_key_drop_user # -rocksdb_flush_write_bytes # -rocksdb_get_hit_l0 # -rocksdb_get_hit_l1 # -rocksdb_get_hit_l2_and_up # -rocksdb_getupdatessince_calls # -rocksdb_git_date # -rocksdb_git_hash # -rocksdb_iter_bytes_read # +Rocksdb_bloom_filter_prefix_checked # +Rocksdb_bloom_filter_prefix_useful # +Rocksdb_bloom_filter_useful # +Rocksdb_bytes_read # +Rocksdb_bytes_written # +Rocksdb_compact_read_bytes # +Rocksdb_compact_write_bytes # +Rocksdb_compaction_key_drop_new # +Rocksdb_compaction_key_drop_obsolete # +Rocksdb_compaction_key_drop_user # +Rocksdb_flush_write_bytes # +Rocksdb_get_hit_l0 # +Rocksdb_get_hit_l1 # +Rocksdb_get_hit_l2_and_up # +Rocksdb_getupdatessince_calls # +Rocksdb_iter_bytes_read # +Rocksdb_memtable_hit # +Rocksdb_memtable_miss # +Rocksdb_no_file_closes # +Rocksdb_no_file_errors # + rocksdb_manual_compactions_processed # + rocksdb_manual_compactions_running # -rocksdb_memtable_hit # -rocksdb_memtable_miss # -rocksdb_no_file_closes # -rocksdb_no_file_errors # -rocksdb_no_file_opens # -rocksdb_num_iterators # -rocksdb_number_block_not_compressed # -rocksdb_number_db_next # -rocksdb_number_db_next_found # -rocksdb_number_db_prev # -rocksdb_number_db_prev_found # -rocksdb_number_db_seek # -rocksdb_number_db_seek_found # -rocksdb_number_deletes_filtered # -rocksdb_number_keys_read # -rocksdb_number_keys_updated # -rocksdb_number_keys_written # -rocksdb_number_merge_failures # -rocksdb_number_multiget_bytes_read # -rocksdb_number_multiget_get # -rocksdb_number_multiget_keys_read # -rocksdb_number_reseeks_iteration # -rocksdb_number_sst_entry_delete # -rocksdb_number_sst_entry_merge # -rocksdb_number_sst_entry_other # -rocksdb_number_sst_entry_put # -rocksdb_number_sst_entry_singledelete # -rocksdb_number_superversion_acquires # -rocksdb_number_superversion_cleanups # -rocksdb_number_superversion_releases # -rocksdb_row_lock_deadlocks # -rocksdb_row_lock_wait_timeouts # -rocksdb_snapshot_conflict_errors # -rocksdb_stall_l0_file_count_limit_slowdowns # -rocksdb_stall_locked_l0_file_count_limit_slowdowns # -rocksdb_stall_l0_file_count_limit_stops # -rocksdb_stall_locked_l0_file_count_limit_stops # -rocksdb_stall_pending_compaction_limit_stops # -rocksdb_stall_pending_compaction_limit_slowdowns # -rocksdb_stall_memtable_limit_stops # -rocksdb_stall_memtable_limit_slowdowns # -rocksdb_stall_total_stops # -rocksdb_stall_total_slowdowns # -rocksdb_stall_micros # -rocksdb_wal_bytes # -rocksdb_wal_group_syncs # -rocksdb_wal_synced # -rocksdb_write_other # -rocksdb_write_self # -rocksdb_write_timedout # -rocksdb_write_wal # +Rocksdb_no_file_opens # +Rocksdb_num_iterators # +Rocksdb_number_block_not_compressed # +Rocksdb_number_db_next # +Rocksdb_number_db_next_found # +Rocksdb_number_db_prev # +Rocksdb_number_db_prev_found # +Rocksdb_number_db_seek # +Rocksdb_number_db_seek_found # +Rocksdb_number_deletes_filtered # +Rocksdb_number_keys_read # +Rocksdb_number_keys_updated # +Rocksdb_number_keys_written # +Rocksdb_number_merge_failures # +Rocksdb_number_multiget_bytes_read # +Rocksdb_number_multiget_get # +Rocksdb_number_multiget_keys_read # +Rocksdb_number_reseeks_iteration # +Rocksdb_number_sst_entry_delete # +Rocksdb_number_sst_entry_merge # +Rocksdb_number_sst_entry_other # +Rocksdb_number_sst_entry_put # +Rocksdb_number_sst_entry_singledelete # +Rocksdb_number_superversion_acquires # +Rocksdb_number_superversion_cleanups # +Rocksdb_number_superversion_releases # +Rocksdb_row_lock_deadlocks # +Rocksdb_row_lock_wait_timeouts # +Rocksdb_snapshot_conflict_errors # +Rocksdb_stall_l0_file_count_limit_slowdowns # +Rocksdb_stall_locked_l0_file_count_limit_slowdowns # +Rocksdb_stall_l0_file_count_limit_stops # +Rocksdb_stall_locked_l0_file_count_limit_stops # +Rocksdb_stall_pending_compaction_limit_stops # +Rocksdb_stall_pending_compaction_limit_slowdowns # +Rocksdb_stall_memtable_limit_stops # +Rocksdb_stall_memtable_limit_slowdowns # +Rocksdb_stall_total_stops # +Rocksdb_stall_total_slowdowns # +Rocksdb_stall_micros # +Rocksdb_wal_bytes # +Rocksdb_wal_group_syncs # +Rocksdb_wal_synced # +Rocksdb_write_other # +Rocksdb_write_self # +Rocksdb_write_timedout # +Rocksdb_write_wal # select VARIABLE_NAME from INFORMATION_SCHEMA.global_status where VARIABLE_NAME LIKE 'rocksdb%'; VARIABLE_NAME ROCKSDB_ROWS_DELETED @@@ -1617,7 -1614,11 +1631,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES @@@ -1726,7 -1729,11 +1744,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result index 24466d9,0b1d346..b096e42 --- a/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result @@@ -1,11 -1,11 +1,11 @@@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] DROP TABLE IF EXISTS t1; +connection slave; include/stop_slave.inc + create table t1 (a int, b int, primary key (a), unique key (b)) engine=rocksdb; +connection master; - create table t1 (a int) engine=rocksdb; +connection slave; show variables like 'rpl_skip_tx_api'; Variable_name Value rpl_skip_tx_api ON diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test index d715eb7,522ff9f..4f34cbe --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test @@@ -1,6 -1,6 +1,7 @@@ --source include/have_rocksdb.inc --source include/have_debug.inc +--source include/have_partition.inc + --source include/not_valgrind.inc --disable_warnings drop table if exists t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test index 5eac859,be8b9fb..61a10a6 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test @@@ -66,9 -63,7 +66,9 @@@ ALTER TABLE t1 ADD INDEX kb(b), ALGORIT --disable_warnings # now do same index using copy algorithm # hitting max row locks (1M) +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; - --error ER_RDB_STATUS_GENERAL + --error ER_GET_ERRMSG ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; diff --cc storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test index 86a0a7f,238e62a..f4da0b7 --- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test @@@ -117,12 -115,12 +117,35 @@@ INSERT INTO t0 VALUES(0) ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +--echo #--------------------------------------------------------------- +--echo # MDEV-16703 Assertion failed in load_auto_incr_value_from_index +--echo #--------------------------------------------------------------- + +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++ ++--echo #---------------------------------- ++--echo # Issue #792 Crash in autoincrement ++--echo #---------------------------------- ++ ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++ ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; ++ + --echo #---------------------------------- + --echo # Issue #869 Crash in autoincrement + --echo #---------------------------------- + + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test index 9e7c816,c1cdf53..53bfba3 --- a/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test @@@ -8,7 -4,8 +8,10 @@@ let $restart_file= $MYSQLTEST_VARDIR/tm let $error_log= $MYSQLTEST_VARDIR/log/my_restart.err; select variable_name, variable_value from information_schema.global_variables where variable_name="rocksdb_ignore_unknown_options"; ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" +--exec find $MYSQLD_DATADIR/#rocksdb/OPTIONS* | sort -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++ --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --shutdown_server 10 diff --cc storage/rocksdb/mysql-test/rocksdb/t/collation.test index 7152925,38ce1b2..3b808bc --- a/storage/rocksdb/mysql-test/rocksdb/t/collation.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/collation.test @@@ -1,13 -1,7 +1,9 @@@ --source include/have_rocksdb.inc ---source include/have_fullregex.inc +# MariaDB doesn't have server variables to check for GCC version, so the +# following check is commented out: +# --source include/have_fullregex.inc - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - - --disable_warnings - DROP TABLE IF EXISTS t1; - --enable_warnings + call mtr.add_suppression("Invalid pattern"); # ci non-indexed column is allowed CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; diff --cc storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 67dae5d,2ae0b40..09998b9 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@@ -28,10 -17,10 +28,11 @@@ select VALUE into @keysIn from INFORMAT CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --cc storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test index b884738,cf8fb29..f4fdaa7 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test @@@ -1,5 -1,5 +1,6 @@@ --source include/have_rocksdb.inc +--source include/have_partition.inc + --source suite/rocksdb/include/have_write_committed.inc # # RocksDB Storage Engine tests diff --cc storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test index e9dcc60,d9ffb08..9f1e28d --- a/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test @@@ -43,41 -46,47 +45,44 @@@ shutdown_server 10 shutdown_server 10; # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message ---exec echo "Expect errors that we are missing two .frm files" ---let SEARCH_FILE=$LOG ---let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t1 is registered in RocksDB but does not have a .frm file +--echo "Expect errors that we are missing two .frm files" +--let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN=Schema mismatch --source include/search_pattern_in_file.inc + --let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t2 is registered in RocksDB but does not have a .frm file + --source include/search_pattern_in_file.inc # Now shut down again and rename one the .frm file back and make a copy of it --exec echo "wait" >$_expect_file_name shutdown_server 10; - + --remove_file $LOG - # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm ---exec cp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm +--copy_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message for the second case ---exec echo "Expect an error that we have an extra .frm file" +--echo "Expect an error that we have an extra .frm file" - --let SEARCH_PATTERN=Schema mismatch + --let SEARCH_FILE=$LOG + --let SEARCH_PATTERN=Schema mismatch - A .frm file exists for table test.t1_dummy, but that table is not registered in RocksDB --source include/search_pattern_in_file.inc # Shut down an clean up diff --cc storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc index 5e5961f,492b5cd..16ad535 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc @@@ -40,9 -40,9 +40,9 @@@ select * from mysql.slave_gtid_info --write_file $MYSQL_TMP_DIR/truncate_tail_wal.sh #!/bin/bash -F=`ls -t $slave_data_dir/.rocksdb/*.log | head -n 1` +F=`ls -t $slave_data_dir/\#rocksdb/*.log | head -n 1` SIZE=`stat -c %s $F` - NEW_SIZE=`expr $SIZE - 10` + NEW_SIZE=`expr $SIZE - 30` truncate -s $NEW_SIZE $F rc=$? if [[ $rc != 0 ]]; then diff --cc storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test index 39688e6,31b50aa..8d3bab4 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test @@@ -1,7 -1,21 +1,21 @@@ --source include/have_rocksdb.inc + CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO valid_values VALUES(65536); + INSERT INTO valid_values VALUES(1024); + INSERT INTO valid_values VALUES(1*1024*1024); + + CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO invalid_values VALUES('\'aaa\''); + INSERT INTO invalid_values VALUES('\'bbb\''); + INSERT INTO invalid_values VALUES('\'-1\''); + INSERT INTO invalid_values VALUES('\'101\''); + INSERT INTO invalid_values VALUES('\'484436\''); + --let $sys_var=ROCKSDB_BLOCK_CACHE_SIZE - --let $read_only=1 + --let $read_only=0 --let $session=0 ---source ../include/rocksdb_sys_var.inc +--source include/rocksdb_sys_var.inc + DROP TABLE valid_values; + DROP TABLE invalid_values; diff --cc storage/rocksdb/rdb_i_s.cc index de33d69,147ad7f..9192795 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@@ -1757,10 -1869,26 +1906,26 @@@ struct st_maria_plugin rdb_i_s_ddl = nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ + 0, /* flags */ + }; + + struct st_mysql_plugin rdb_i_s_sst_props = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_SST_PROPS", + "Facebook", + "RocksDB SST Properties", + PLUGIN_LICENSE_GPL, + rdb_i_s_sst_props_init, + rdb_i_s_deinit, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; -struct st_mysql_plugin rdb_i_s_index_file_map = { +struct st_maria_plugin rdb_i_s_index_file_map = { MYSQL_INFORMATION_SCHEMA_PLUGIN, &rdb_i_s_info, "ROCKSDB_INDEX_FILE_MAP", diff --cc storage/rocksdb/rdb_i_s.h index d6a48bf,dd9f4a9..f4c4b04 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@@ -21,16 -21,17 +21,17 @@@ namespace myrocks Declare INFORMATION_SCHEMA (I_S) plugins needed by MyRocks storage engine. */ -extern struct st_mysql_plugin rdb_i_s_cfstats; -extern struct st_mysql_plugin rdb_i_s_dbstats; -extern struct st_mysql_plugin rdb_i_s_perf_context; -extern struct st_mysql_plugin rdb_i_s_perf_context_global; -extern struct st_mysql_plugin rdb_i_s_cfoptions; -extern struct st_mysql_plugin rdb_i_s_compact_stats; -extern struct st_mysql_plugin rdb_i_s_global_info; -extern struct st_mysql_plugin rdb_i_s_ddl; -extern struct st_mysql_plugin rdb_i_s_sst_props; -extern struct st_mysql_plugin rdb_i_s_index_file_map; -extern struct st_mysql_plugin rdb_i_s_lock_info; -extern struct st_mysql_plugin rdb_i_s_trx_info; -extern struct st_mysql_plugin rdb_i_s_deadlock_info; +extern struct st_maria_plugin rdb_i_s_cfstats; +extern struct st_maria_plugin rdb_i_s_dbstats; +extern struct st_maria_plugin rdb_i_s_perf_context; +extern struct st_maria_plugin rdb_i_s_perf_context_global; +extern struct st_maria_plugin rdb_i_s_cfoptions; +extern struct st_maria_plugin rdb_i_s_compact_stats; +extern struct st_maria_plugin rdb_i_s_global_info; +extern struct st_maria_plugin rdb_i_s_ddl; ++extern struct st_maria_plugin rdb_i_s_sst_props; +extern struct st_maria_plugin rdb_i_s_index_file_map; +extern struct st_maria_plugin rdb_i_s_lock_info; +extern struct st_maria_plugin rdb_i_s_trx_info; +extern struct st_maria_plugin rdb_i_s_deadlock_info; } // namespace myrocks diff --cc storage/rocksdb/rocksdb index ba295cd,0000000..926f3a7 mode 160000,000000..160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@@ -1,1 -1,0 +1,1 @@@ - Subproject commit ba295cda29daee3ffe58549542804efdfd969784 ++Subproject commit 926f3a78a64b327475ee6c60b6c8ab4f34253204