revision-id: dbe73588cd82dc6b6547425e7233b7899f61dfd5 parent(s): e44ca6cc9c300cbdf93c64110bd8cf2be8125379 03edf2ed04dbffe8c413fe0dd2715684e1627371 committer: Sergei Petrunia branch nick: 10.2-r11 timestamp: 2018-05-07 21:38:18 +0300 message: Merge branch 'bb-10.2-mariarocks-merge' of github.com:MariaDB/server into 10.2 Manually resolved the conflicts sql/mysqld.h | 2 +- storage/rocksdb/CMakeLists.txt | 8 +- storage/rocksdb/build_rocksdb.cmake | 9 +- storage/rocksdb/event_listener.cc | 10 + storage/rocksdb/event_listener.h | 3 + storage/rocksdb/ha_rocksdb.cc | 1887 ++++++++++++++------ storage/rocksdb/ha_rocksdb.h | 101 +- .../rocksdb/include/autoinc_crash_safe.inc | 150 ++ .../mysql-test/rocksdb/include/bulk_load.inc | 156 ++ .../rocksdb/include/bulk_load_unsorted.inc | 144 ++ .../include/restart_mysqld_with_invalid_option.inc | 8 + .../rocksdb/include/start_mysqld_with_option.inc | 14 + .../r/add_index_inplace_sstfilewriter.result | 4 + .../rocksdb/r/allow_no_primary_key_with_sk.result | 17 + .../r/allow_to_start_after_corruption.result | 38 + .../mysql-test/rocksdb/r/autoinc_crash_safe.result | 132 ++ .../rocksdb/r/autoinc_crash_safe_partition.result | 132 ++ .../mysql-test/rocksdb/r/autoinc_debug.result | 107 ++ .../mysql-test/rocksdb/r/autoinc_vars.result | 79 + .../mysql-test/rocksdb/r/autoincrement.result | 1 - .../mysql-test/rocksdb/r/bloomfilter5.result | 62 + .../rocksdb/mysql-test/rocksdb/r/bulk_load.result | 7 +- .../rocksdb/r/bulk_load_drop_table.result | 11 + .../mysql-test/rocksdb/r/bulk_load_errors.result | 64 +- .../mysql-test/rocksdb/r/bulk_load_rev_cf.result | 7 +- .../rocksdb/r/bulk_load_rev_cf_and_data.result | 7 +- .../mysql-test/rocksdb/r/bulk_load_rev_data.result | 7 +- .../mysql-test/rocksdb/r/bulk_load_unsorted.result | 59 +- .../rocksdb/r/bulk_load_unsorted_rev.result | 120 ++ .../mysql-test/rocksdb/r/cardinality.result | 35 + .../rocksdb/r/check_ignore_unknown_options.result | 7 + .../mysql-test/rocksdb/r/deadlock_tracking.result | 67 +- .../rocksdb/mysql-test/rocksdb/r/i_s_ddl.result | 19 +- .../mysql-test/rocksdb/r/i_s_deadlock.result | 215 +++ .../rocksdb/r/index_merge_rocksdb.result | 2 +- .../mysql-test/rocksdb/r/information_schema.result | 7 +- .../rocksdb/mysql-test/rocksdb/r/issue255.result | 47 + .../rocksdb/r/lock_wait_timeout_stats.result | 8 + .../mysql-test/rocksdb/r/mariadb_port_fixes.result | 1 + .../mysql-test/rocksdb/r/max_open_files.result | 21 + .../rocksdb/r/optimizer_loose_index_scans.result | 42 +- .../mysql-test/rocksdb/r/perf_context.result | 32 +- .../rocksdb/mysql-test/rocksdb/r/rocksdb.result | 90 +- .../mysql-test/rocksdb/r/rocksdb_debug.result | 11 + .../mysql-test/rocksdb/r/rocksdb_range2.result | 2 +- .../mysql-test/rocksdb/r/show_engine.result | 36 +- .../rocksdb/r/skip_validate_tmp_table.result | 18 +- .../rocksdb/r/ttl_primary_read_filtering.result | 28 + .../mysql-test/rocksdb/r/type_varchar.result | 14 +- .../rocksdb/r/use_direct_reads_writes.result | 32 +- .../rocksdb/mysql-test/rocksdb/r/write_sync.result | 5 +- .../rocksdb/t/add_index_inplace_sstfilewriter.test | 5 + .../rocksdb/t/allow_no_primary_key_with_sk.test | 12 + .../rocksdb/t/allow_to_start_after_corruption.test | 75 + .../mysql-test/rocksdb/t/autoinc_crash_safe.cnf | 8 + .../mysql-test/rocksdb/t/autoinc_crash_safe.test | 9 + .../rocksdb/t/autoinc_crash_safe_partition.cnf | 8 + .../rocksdb/t/autoinc_crash_safe_partition.test | 10 + .../mysql-test/rocksdb/t/autoinc_debug-master.opt | 1 + .../mysql-test/rocksdb/t/autoinc_debug.test | 118 ++ .../rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 38 + .../mysql-test/rocksdb/t/autoincrement.test | 3 - .../mysql-test/rocksdb/t/bloomfilter5-master.opt | 1 + .../rocksdb/mysql-test/rocksdb/t/bloomfilter5.test | 61 + storage/rocksdb/mysql-test/rocksdb/t/bulk_load.inc | 156 -- .../rocksdb/mysql-test/rocksdb/t/bulk_load.test | 2 +- .../mysql-test/rocksdb/t/bulk_load_drop_table.test | 19 + .../mysql-test/rocksdb/t/bulk_load_errors.test | 97 +- .../mysql-test/rocksdb/t/bulk_load_rev_cf.test | 2 +- .../rocksdb/t/bulk_load_rev_cf_and_data.test | 2 +- .../mysql-test/rocksdb/t/bulk_load_rev_data.test | 2 +- .../mysql-test/rocksdb/t/bulk_load_unsorted.test | 134 +- .../rocksdb/t/bulk_load_unsorted_rev.test | 5 + .../rocksdb/mysql-test/rocksdb/t/cardinality.test | 42 + .../rocksdb/t/check_ignore_unknown_options.test | 22 + .../mysql-test/rocksdb/t/deadlock_tracking.test | 20 +- storage/rocksdb/mysql-test/rocksdb/t/i_s_ddl.test | 7 +- .../rocksdb/mysql-test/rocksdb/t/i_s_deadlock.test | 158 ++ .../mysql-test/rocksdb/t/index_merge_rocksdb.test | 7 +- .../mysql-test/rocksdb/t/index_merge_rocksdb2.test | 4 +- .../mysql-test/rocksdb/t/information_schema.test | 4 +- .../rocksdb/t/insert_optimized_config-master.opt | 1 + storage/rocksdb/mysql-test/rocksdb/t/issue255.test | 32 + .../rocksdb/t/lock_wait_timeout_stats.test | 4 + .../mysql-test/rocksdb/t/max_open_files.test | 53 + .../rocksdb/mysql-test/rocksdb/t/mysqldump.test | 8 +- .../rocksdb/mysql-test/rocksdb/t/mysqldump2.test | 2 +- storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test | 5 +- .../mysql-test/rocksdb/t/rocksdb_debug.test | 14 + .../mysql-test/rocksdb/t/rocksdb_range2.test | 1 + .../rocksdb/t/skip_validate_tmp_table.test | 38 +- .../rocksdb/t/ttl_primary_read_filtering.test | 19 +- .../rocksdb/mysql-test/rocksdb/t/type_varchar.test | 15 +- .../rocksdb/t/use_direct_reads_writes.test | 64 +- .../rocksdb/mysql-test/rocksdb/t/write_sync.test | 9 +- .../include/rpl_no_unique_check_on_lag.inc | 1 + .../r/singledelete_idempotent_recovery.result | 1 + .../mysql-test/rocksdb_rpl/t/multiclient_2pc.test | 1 + .../t/singledelete_idempotent_recovery.test | 6 + .../rocksdb_stress/r/rocksdb_stress.result | 2 + .../rocksdb_stress/r/rocksdb_stress_crash.result | 2 + .../mysql-test/rocksdb_stress/t/load_generator.py | 13 + .../rocksdb_stress/t/rocksdb_stress.test | 2 + .../rocksdb_stress/t/rocksdb_stress_crash.test | 2 + ...db_allow_to_start_after_corruption_basic.result | 7 + .../r/rocksdb_bytes_per_sync_basic.result | 84 +- .../r/rocksdb_concurrent_prepare_basic.result | 14 - .../rocksdb_flush_memtable_on_analyze_basic.result | 58 - .../r/rocksdb_ignore_unknown_options_basic.result | 14 + .../r/rocksdb_max_open_files_basic.result | 10 +- .../r/rocksdb_max_row_locks_basic.result | 18 +- .../r/rocksdb_two_write_queues_basic.result | 14 + .../r/rocksdb_update_cf_options.result | 38 + .../r/rocksdb_update_cf_options_basic.result | 18 +- .../r/rocksdb_wal_bytes_per_sync_basic.result | 84 +- ...ksdb_allow_to_start_after_corruption_basic.test | 6 + .../t/rocksdb_bytes_per_sync_basic.test | 17 +- .../t/rocksdb_concurrent_prepare_basic.test | 16 - .../t/rocksdb_flush_memtable_on_analyze_basic.test | 46 - .../t/rocksdb_ignore_unknown_options_basic.test | 16 + .../t/rocksdb_max_open_files_basic.test | 10 +- .../t/rocksdb_two_write_queues_basic.test | 16 + .../t/rocksdb_update_cf_options.test | 22 + .../t/rocksdb_update_cf_options_basic.test | 17 +- .../t/rocksdb_wal_bytes_per_sync_basic.test | 18 +- storage/rocksdb/patch/port/win/io_win.h | 446 ----- storage/rocksdb/properties_collector.cc | 145 +- storage/rocksdb/properties_collector.h | 41 +- storage/rocksdb/rdb_cf_options.cc | 8 + storage/rocksdb/rdb_cf_options.h | 3 + storage/rocksdb/rdb_compact_filter.h | 2 +- storage/rocksdb/rdb_datadic.cc | 196 +- storage/rocksdb/rdb_datadic.h | 161 +- storage/rocksdb/rdb_i_s.cc | 149 +- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_io_watchdog.cc | 2 +- storage/rocksdb/rdb_perf_context.cc | 31 +- storage/rocksdb/rdb_perf_context.h | 14 + storage/rocksdb/rdb_psi.cc | 3 +- storage/rocksdb/rdb_psi.h | 3 +- storage/rocksdb/rdb_sst_info.cc | 38 +- storage/rocksdb/rdb_sst_info.h | 6 +- storage/rocksdb/rdb_utils.cc | 33 + storage/rocksdb/rdb_utils.h | 12 +- storage/rocksdb/rocksdb | 2 +- 145 files changed, 5356 insertions(+), 1895 deletions(-) diff --cc sql/mysqld.h index 74e4f1e,a2a97d0..5a2dc6d --- a/sql/mysqld.h +++ b/sql/mysqld.h @@@ -208,8 -208,8 +208,8 @@@ extern uint slave_net_timeout extern int max_user_connections; extern volatile ulong cached_thread_count; extern ulong what_to_log,flush_time; -extern ulong max_prepared_stmt_count, prepared_stmt_count; +extern uint max_prepared_stmt_count, prepared_stmt_count; - extern ulong open_files_limit; + extern MYSQL_PLUGIN_IMPORT ulong open_files_limit; extern ulonglong binlog_cache_size, binlog_stmt_cache_size; extern ulonglong max_binlog_cache_size, max_binlog_stmt_cache_size; extern ulong max_binlog_size; diff --cc storage/rocksdb/ha_rocksdb.cc index 03086db,e7abac7..baada70 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@@ -1810,16 -1846,31 +1857,39 @@@ protected virtual rocksdb::Iterator * get_iterator(const rocksdb::ReadOptions &options, rocksdb::ColumnFamilyHandle *column_family) = 0; + +protected: + /* + The following two are helper functions to be overloaded by child classes. + They should provide RocksDB's savepoint semantics. + */ + virtual void do_set_savepoint() = 0; + virtual void do_rollback_to_savepoint() = 0; - public: + /* + @detail + This function takes in the WriteBatch of the transaction to add + all the AUTO_INCREMENT merges. It does so by iterating through + m_auto_incr_map and then constructing key/value pairs to call merge upon. + + @param wb + */ + rocksdb::Status merge_auto_incr_map(rocksdb::WriteBatchBase *const wb) { + DBUG_EXECUTE_IF("myrocks_autoinc_upgrade", return rocksdb::Status::OK();); + + // Iterate through the merge map merging all keys into data dictionary. + rocksdb::Status s; + for (auto &it : m_auto_incr_map) { + s = dict_manager.put_auto_incr_val(wb, it.first, it.second); + if (!s.ok()) { + return s; + } + } + m_auto_incr_map.clear(); + return s; + } + + public: const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; #ifdef MARIAROCKS_NOT_YET @@@ -4049,17 -4219,24 +4316,33 @@@ bool prevent_myrocks_loading= false */ static int rocksdb_init_func(void *const p) { + DBUG_ENTER_FUNC(); + if (prevent_myrocks_loading) + { + my_error(ER_INTERNAL_ERROR, MYF(0), + "Loading MyRocks plugin after it has been unloaded is not " + "supported. Please restart mysqld"); + DBUG_RETURN(1); + } + + if (rdb_check_rocksdb_corruption()) { + sql_print_error("RocksDB: There was a corruption detected in RockDB files. " + "Check error log emitted earlier for more details."); + if (rocksdb_allow_to_start_after_corruption) { + sql_print_information( + "RocksDB: Remove rocksdb_allow_to_start_after_corruption to prevent " + "server operating if RocksDB corruption is detected."); + } else { + sql_print_error("RocksDB: The server will exit normally and stop restart " + "attempts. Remove %s file from data directory and " + "start mysqld manually.", + rdb_corruption_marker_file_name().c_str()); + exit(0); + } + } + // Validate the assumption about the size of ROCKSDB_SIZEOF_HIDDEN_PK_COLUMN. static_assert(sizeof(longlong) == 8, "Assuming that longlong is 8 bytes."); @@@ -12655,20 -13218,14 +13385,26 @@@ double ha_rocksdb::read_time(uint index DBUG_RETURN((rows / 20.0) + 1); } + std::string rdb_corruption_marker_file_name() { + std::string ret(rocksdb_datadir); + ret.append("/ROCKSDB_CORRUPTED"); + return ret; + } + +void sql_print_verbose_info(const char *format, ...) +{ + va_list args; + + if (global_system_variables.log_warnings > 2) { + va_start(args, format); + sql_print_information_v(format, args); + va_end(args); + } +} + } // namespace myrocks + /** Construct and emit duplicate key error message using information from table's record buffer. diff --cc storage/rocksdb/ha_rocksdb.h index d929ca1,8e4434e..0b008a6 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@@ -1412,11 -1458,10 +1458,14 @@@ private Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; + // file name indicating RocksDB data corruption + std::string rdb_corruption_marker_file_name(); + -const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA; +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_GAMMA; +extern bool prevent_myrocks_loading; + +void sql_print_verbose_info(const char *format, ...); } // namespace myrocks + diff --cc storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 9674b2b,8582b83..98c5ebe --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@@ -69,22 -69,16 +69,23 @@@ set global rocksdb_strict_collation_che # select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; plugin_name plugin_maturity -ROCKSDB Beta -ROCKSDB_CFSTATS Beta -ROCKSDB_DBSTATS Beta -ROCKSDB_PERF_CONTEXT Beta -ROCKSDB_PERF_CONTEXT_GLOBAL Beta -ROCKSDB_CF_OPTIONS Beta -ROCKSDB_COMPACTION_STATS Beta -ROCKSDB_GLOBAL_INFO Beta -ROCKSDB_DDL Beta -ROCKSDB_INDEX_FILE_MAP Beta -ROCKSDB_LOCKS Beta -ROCKSDB_TRX Beta -ROCKSDB_DEADLOCK Beta +ROCKSDB Gamma +ROCKSDB_CFSTATS Gamma +ROCKSDB_DBSTATS Gamma +ROCKSDB_PERF_CONTEXT Gamma +ROCKSDB_PERF_CONTEXT_GLOBAL Gamma +ROCKSDB_CF_OPTIONS Gamma +ROCKSDB_COMPACTION_STATS Gamma +ROCKSDB_GLOBAL_INFO Gamma +ROCKSDB_DDL Gamma +ROCKSDB_INDEX_FILE_MAP Gamma +ROCKSDB_LOCKS Gamma +ROCKSDB_TRX Gamma ++ROCKSDB_DEADLOCK Gamma +# +# MDEV-12466 : Assertion `thd->transaction.stmt.is_empty() || thd->in_sub_stmt || ... +# +CREATE TABLE t1 (i INT) ENGINE=RocksDB; +FLUSH TABLE t1 FOR EXPORT; +ERROR HY000: Storage engine ROCKSDB of the table `test`.`t1` doesn't have this option +DROP TABLE t1;