- commits - lists.mariadb.org

[Commits] c930afd: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge
by psergey＠askmonty.org 28 Aug '18

28 Aug '18

revision-id: c930afd47eb68aaa5a6a1000969cef881d3fbb0b parent(s): 55163ba1bdb1a05daadc66c41c959994231b361c faa4d8f8c6c12c09541c4733773ac700c5e643cf committer: Sergei Petrunia branch nick: 10.2-mariarocks timestamp: 2018-08-28 14:09:04 +0300 message: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge Move up-to this revision in the upstream: commit de1e8c7bfe7c875ea284b55040e8f3cd3a56fcc2 Author: Abhinav Sharma <abhinavsharma(a)fb.com> Date: Thu Aug 23 14:34:39 2018 -0700 Log updates to semi-sync whitelist in the error log Summary: Plugin variable changes are not logged in the error log even when log_global_var_changes is enabled. Logging updates to whitelist will help in debugging. Reviewed By: guokeno0 Differential Revision: D9483807 fbshipit-source-id: e111cda773d storage/rocksdb/CMakeLists.txt | 8 +- storage/rocksdb/build_rocksdb.cmake | 6 + storage/rocksdb/ha_rocksdb.cc | 1235 +++++++++++++++----- storage/rocksdb/ha_rocksdb.h | 53 +- storage/rocksdb/ha_rocksdb_proto.h | 7 +- storage/rocksdb/mysql-test/rocksdb/combinations | 6 + .../rocksdb/include/have_write_committed.inc | 3 + .../mysql-test/rocksdb/r/add_index_inplace.result | 24 + .../r/add_index_inplace_sstfilewriter.result | 2 +- .../rocksdb/r/allow_no_primary_key.result | 6 +- .../mysql-test/rocksdb/r/autoinc_debug.result | 40 +- .../mysql-test/rocksdb/r/autoinc_vars.result | 18 + .../mysql-test/rocksdb/r/bloomfilter5.result | 4 +- .../rocksdb/r/bloomfilter_bulk_load.result | 15 + .../mysql-test/rocksdb/r/bulk_load_sk.result | 229 ++++ .../mysql-test/rocksdb/r/cardinality.result | 17 +- .../rocksdb/mysql-test/rocksdb/r/collation.result | 20 +- .../rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result | 21 + .../rocksdb/r/create_no_primary_key_table.result | 38 + .../mysql-test/rocksdb/r/ddl_high_priority.result | 71 +- .../mysql-test/rocksdb/r/deadlock_tracking.result | 57 + .../rocksdb/mysql-test/rocksdb/r/drop_table.result | 11 + .../mysql-test/rocksdb/r/explicit_snapshot.result | 265 +++++ .../mysql-test/rocksdb/r/i_s_deadlock.result | 55 +- .../mysql-test/rocksdb/r/index_file_map.result | 3 + .../rocksdb/r/index_merge_rocksdb2.result | 52 +- .../mysql-test/rocksdb/r/information_schema.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/lock.result | 15 + .../rocksdb/mysql-test/rocksdb/r/mysqldump.result | 5 +- .../r/percona_nonflushing_analyze_debug.result | 19 + .../rocksdb/r/prefix_extractor_override.result | 7 +- .../mysql-test/rocksdb/r/read_only_tx.result | 2 +- .../rocksdb/mysql-test/rocksdb/r/rocksdb.result | 24 +- .../mysql-test/rocksdb/r/rocksdb_checksums.result | 6 +- .../rocksdb/r/rocksdb_deadlock_detect_rc.result | 16 +- .../rocksdb/r/rocksdb_deadlock_detect_rr.result | 16 +- .../rocksdb/r/secondary_key_update_lock.result | 11 + .../mysql-test/rocksdb/r/show_engine.result | 53 + .../mysql-test/rocksdb/r/transaction.result | 17 + .../mysql-test/rocksdb/r/trx_info_rpl.result | 2 +- storage/rocksdb/mysql-test/rocksdb/r/update.result | 8 + .../mysql-test/rocksdb/r/use_direct_reads.result | 18 + .../mysql-test/rocksdb/t/add_index_inplace.test | 18 + .../rocksdb/t/add_index_inplace_crash.test | 1 + .../rocksdb/t/add_index_inplace_sstfilewriter.test | 2 +- .../mysql-test/rocksdb/t/autoinc_debug.test | 17 +- .../rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 23 + .../mysql-test/rocksdb/t/bloomfilter5-master.opt | 4 +- .../rocksdb/t/bloomfilter_bulk_load-master.opt | 2 + .../rocksdb/t/bloomfilter_bulk_load.test | 35 + .../rocksdb/mysql-test/rocksdb/t/bulk_load_sk.test | 119 ++ .../rocksdb/mysql-test/rocksdb/t/cardinality.test | 21 +- .../rocksdb/t/check_ignore_unknown_options.test | 3 + .../rocksdb/mysql-test/rocksdb/t/collation.test | 36 +- .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.cnf | 4 + .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.test | 87 ++ .../t/create_no_primary_key_table-master.opt | 1 + .../rocksdb/t/create_no_primary_key_table.test | 44 + .../mysql-test/rocksdb/t/deadlock_tracking.test | 76 +- .../rocksdb/mysql-test/rocksdb/t/drop_table.test | 27 + .../rocksdb/t/explicit_snapshot-master.opt | 1 + .../mysql-test/rocksdb/t/explicit_snapshot.test | 263 +++++ .../rocksdb/mysql-test/rocksdb/t/i_s_deadlock.test | 14 +- .../mysql-test/rocksdb/t/index_file_map.test | 3 + .../rocksdb/t/index_merge_rocksdb2-master.opt | 2 +- .../mysql-test/rocksdb/t/information_schema.test | 1 + .../rocksdb/t/insert_optimized_config-master.opt | 8 - .../rocksdb/t/insert_optimized_config.test | 10 + storage/rocksdb/mysql-test/rocksdb/t/lock.test | 22 + .../t/percona_nonflushing_analyze_debug.test | 11 + .../rocksdb/t/prefix_extractor_override.test | 14 +- storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test | 3 +- .../mysql-test/rocksdb/t/rocksdb_checksums.test | 10 +- .../rocksdb/t/rocksdb_deadlock_detect.inc | 35 +- .../rocksdb/t/secondary_key_update_lock.test | 26 + .../mysql-test/rocksdb/t/set_checkpoint.inc | 2 +- .../rocksdb/mysql-test/rocksdb/t/show_engine.test | 13 + .../rocksdb/mysql-test/rocksdb/t/transaction.test | 23 + .../rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test | 4 +- storage/rocksdb/mysql-test/rocksdb/t/update.test | 10 + .../mysql-test/rocksdb/t/use_direct_reads.test | 37 + .../mysql-test/rocksdb/t/validate_datadic.test | 16 +- .../rocksdb/mysql-test/rocksdb_rpl/combinations | 8 +- .../r/rpl_missing_columns_sk_update.result | 62 + .../rpl_mts_dependency_unique_key_conflicts.result | 44 + .../rocksdb_rpl/r/rpl_rocksdb_snapshot.result | 4 +- .../t/rpl_gtid_crash_safe_wal_corrupt.inc | 2 +- .../t/rpl_missing_columns_sk_update.cnf | 13 + .../t/rpl_missing_columns_sk_update.test | 69 ++ .../t/rpl_mts_dependency_unique_key_conflicts.test | 64 + .../t/rpl_rocksdb_2pc_crash_recover.test | 1 + .../rocksdb/mysql-test/rocksdb_stress/combinations | 5 + .../r/rocksdb_block_cache_size_basic.result | 84 +- .../r/rocksdb_bulk_load_allow_sk_basic.result | 100 ++ ...sdb_commit_time_batch_for_recovery_basic.result | 121 ++ .../r/rocksdb_create_checkpoint_basic.result | 2 +- ...ksdb_debug_manual_compaction_delay_basic.result | 46 + ...ksdb_error_on_suboptimal_collation_basic.result | 7 + .../rocksdb_manual_compaction_threads_basic.result | 93 ++ .../r/rocksdb_max_manifest_file_size_basic.result | 2 +- .../r/rocksdb_max_manual_compactions_basic.result | 57 + .../r/rocksdb_max_row_locks_basic.result | 21 + .../r/rocksdb_stats_recalc_rate_basic.result | 53 + .../r/rocksdb_write_batch_max_bytes_basic.result | 4 +- .../r/rocksdb_write_policy_basic.result | 15 + .../t/rocksdb_block_cache_size_basic.test | 16 +- .../t/rocksdb_bulk_load_allow_sk_basic.test | 18 + ...cksdb_commit_time_batch_for_recovery_basic.test | 20 + .../t/rocksdb_create_checkpoint_basic.test | 2 +- ...ocksdb_debug_manual_compaction_delay_basic.test | 16 + ...ocksdb_error_on_suboptimal_collation_basic.test | 6 + .../t/rocksdb_manual_compaction_threads_basic.test | 17 + .../t/rocksdb_max_manual_compactions_basic.test | 17 + .../t/rocksdb_max_row_locks_basic.test | 1 + .../t/rocksdb_stats_recalc_rate_basic.test | 17 + .../t/rocksdb_write_batch_max_bytes_basic.test | 4 +- .../t/rocksdb_write_policy_basic.test | 17 + storage/rocksdb/properties_collector.cc | 61 +- storage/rocksdb/properties_collector.h | 3 +- storage/rocksdb/rdb_cf_options.cc | 2 - storage/rocksdb/rdb_comparator.h | 46 +- storage/rocksdb/rdb_datadic.cc | 6 +- storage/rocksdb/rdb_datadic.h | 5 +- storage/rocksdb/rdb_i_s.cc | 167 ++- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_index_merge.cc | 5 + storage/rocksdb/rdb_perf_context.cc | 2 - storage/rocksdb/rdb_psi.cc | 17 +- storage/rocksdb/rdb_psi.h | 10 +- storage/rocksdb/rdb_sst_info.cc | 4 +- storage/rocksdb/rdb_threads.h | 31 +- storage/rocksdb/rdb_utils.cc | 23 - storage/rocksdb/rocksdb | 2 +- 133 files changed, 4308 insertions(+), 639 deletions(-) diff --cc storage/rocksdb/CMakeLists.txt index 2a4ea4d,54b0b1f..2689dc7 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@@ -130,55 -53,64 +130,58 @@@ if(HAVE_THREAD_LOCAL ADD_DEFINITIONS(-DROCKSDB_SUPPORT_THREAD_LOCAL) endif() -SET(ROCKSDB_SOURCES - ha_rocksdb.cc ha_rocksdb.h ha_rocksdb_proto.h +INCLUDE(build_rocksdb.cmake) + +ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib + ha_rocksdb_proto.h logger.h rdb_comparator.h - rdb_datadic.cc rdb_datadic.h - rdb_cf_options.cc rdb_cf_options.h - rdb_cf_manager.cc rdb_cf_manager.h - properties_collector.cc properties_collector.h - event_listener.cc event_listener.h - rdb_i_s.cc rdb_i_s.h - rdb_index_merge.cc rdb_index_merge.h - rdb_io_watchdog.cc rdb_io_watchdog.h - rdb_perf_context.cc rdb_perf_context.h - rdb_mutex_wrapper.cc rdb_mutex_wrapper.h - rdb_psi.h rdb_psi.cc - rdb_sst_info.cc rdb_sst_info.h - rdb_utils.cc rdb_utils.h rdb_buff.h - rdb_threads.cc rdb_threads.h - ${ROCKSDB_LIB_SOURCES} + rdb_cf_options.cc + rdb_cf_options.h + event_listener.cc + event_listener.h + rdb_perf_context.cc + rdb_perf_context.h + rdb_buff.h + rdb_mariadb_port.h ) -IF(WITH_FB_TSAN) - SET(PIC_EXT "_pic") -ELSE() - SET(PIC_EXT "") -ENDIF() +ADD_DEPENDENCIES(rocksdb_aux_lib GenError) -SET(rocksdb_static_libs ) -IF (WITH_SNAPPY) - FIND_LIBRARY(SNAPPY_LIBRARY - NAMES libsnappy${PIC_EXT}.a snappy - HINTS ${WITH_SNAPPY}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${SNAPPY_LIBRARY}) - ADD_DEFINITIONS(-DSNAPPY) -ENDIF() +# MARIAROCKS-TODO: how to properly depend on -lrt ? +TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY}) +if (UNIX AND NOT APPLE) + TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt) +endif() -IF (WITH_LZ4) +TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib) + FIND_LIBRARY(LZ4_LIBRARY + NAMES liblz4${PIC_EXT}.a lz4 + HINTS ${WITH_LZ4}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${LZ4_LIBRARY}) - ADD_DEFINITIONS(-DLZ4) -ENDIF() -IF (WITH_BZ2) - FIND_LIBRARY(BZ2_LIBRARY - NAMES libbz2${PIC_EXT}.a bz2 - HINTS ${WITH_BZ2}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${BZ2_LIBRARY}) - ADD_DEFINITIONS(-DBZIP2) +IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + + # MARIAROCKS_NOT_YET: Add -frtti flag when compiling RocksDB files. + # TODO: is this the right way to do this? + # - SQL layer and storage/rocksdb/*.cc are compiled with -fnortti + # - RocksDB files are compiled with "-fnortti ... -frtti" + # - This causes RocksDB headers to be compiled with different settings: + # = with RTTI when compiling RocksDB + # = without RTTI when compiling storage/rocksdb/*.cc + # + # (facebook/mysql-5.6 just compiles everything without -f*rtti, which means + # everything is compiled with -frtti) + # + # (also had to add -frtti above, because something that event_listener.cc + # includes requires it. So, now everything in MariaRocks is compiled with + # -frtti) + set_source_files_properties(event_listener.cc rdb_cf_options.cc rdb_sst_info.cc + PROPERTIES COMPILE_FLAGS -frtti) ENDIF() -IF (WITH_ZSTD) - SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZSTD_LIBRARY}) - ADD_DEFINITIONS(-DZSTD) +CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU) +IF(HAVE_SCHED_GETCPU) + ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1) ENDIF() IF (WITH_TBB) diff --cc storage/rocksdb/build_rocksdb.cmake index 895d473,0000000..f09024e mode 100644,000000..100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@@ -1,417 -1,0 +1,423 @@@ + +if(POLICY CMP0042) + cmake_policy(SET CMP0042 NEW) +endif() + +SET(ROCKSDB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb) + +INCLUDE_DIRECTORIES( + ${CMAKE_CURRENT_BINARY_DIR} + ${ROCKSDB_SOURCE_DIR} + ${ROCKSDB_SOURCE_DIR}/include + ${ROCKSDB_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src +) + +list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_SOURCE_DIR}/cmake/modules/") + +if(WIN32) + # include(${ROCKSDB_SOURCE_DIR}/thirdparty.inc) +else() + option(WITH_ROCKSDB_JEMALLOC "build RocksDB with JeMalloc" OFF) + if(WITH_ROCKSDB_JEMALLOC) + find_package(JeMalloc REQUIRED) + add_definitions(-DROCKSDB_JEMALLOC) + include_directories(${JEMALLOC_INCLUDE_DIR}) + endif() + if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + # FreeBSD has jemaloc as default malloc + add_definitions(-DROCKSDB_JEMALLOC) + set(WITH_JEMALLOC ON) + endif() +endif() + + +# Optional compression libraries. + +foreach(compression_lib LZ4 BZIP2 ZSTD snappy) + FIND_PACKAGE(${compression_lib} QUIET) + + SET(WITH_ROCKSDB_${compression_lib} AUTO CACHE STRING + "Build RocksDB with ${compression_lib} compression. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") + + if(${WITH_ROCKSDB_${compression_lib}} STREQUAL "ON" AND NOT ${${compression_lib}_FOUND}) + MESSAGE(FATAL_ERROR + "${compression_lib} library was not found, but WITH_ROCKSDB${compression_lib} option is ON.\ + Either set WITH_ROCKSDB${compression_lib} to OFF, or make sure ${compression_lib} is installed") + endif() +endforeach() + +if(LZ4_FOUND AND (NOT WITH_ROCKSDB_LZ4 STREQUAL "OFF")) + add_definitions(-DLZ4) + include_directories(${LZ4_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARY}) +endif() + +if(BZIP2_FOUND AND (NOT WITH_ROCKSDB_BZIP2 STREQUAL "OFF")) + add_definitions(-DBZIP2) + include_directories(${BZIP2_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) +endif() + +if(SNAPPY_FOUND AND (NOT WITH_ROCKSDB_SNAPPY STREQUAL "OFF")) + add_definitions(-DSNAPPY) + include_directories(${SNAPPY_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES}) +endif() + +include(CheckFunctionExists) +if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF")) + SET(CMAKE_REQUIRED_LIBRARIES zstd) + CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID) + UNSET(CMAKE_REQUIRED_LIBRARIES) + if (WITH_ROCKSDB_ZSTD STREQUAL "ON" AND NOT ZSTD_VALID) + MESSAGE(FATAL_ERROR + "WITH_ROCKSDB_ZSTD is ON and ZSTD library was found, but the version needs to be >= 1.1.3") + endif() + if (ZSTD_VALID) + add_definitions(-DZSTD) + include_directories(${ZSTD_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY}) + endif() +endif() + +add_definitions(-DZLIB) +list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARY}) + +if(CMAKE_SYSTEM_NAME MATCHES "Cygwin") + add_definitions(-fno-builtin-memcmp -DCYGWIN) +elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") + add_definitions(-DOS_MACOSX) +elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") + add_definitions(-DOS_LINUX) +elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS") + add_definitions(-DOS_SOLARIS) +elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + add_definitions(-DOS_FREEBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + add_definitions(-DOS_NETBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD") + add_definitions(-DOS_OPENBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly") + add_definitions(-DOS_DRAGONFLYBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "Android") + add_definitions(-DOS_ANDROID) +elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") + add_definitions(-DOS_WIN) +endif() + +IF(MSVC) + add_definitions(/wd4244) +ENDIF() +if(NOT WIN32) + add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX) +endif() + +option(WITH_FALLOCATE "build with fallocate" ON) + +if(WITH_FALLOCATE AND UNIX) + include(CheckCSourceCompiles) + CHECK_C_SOURCE_COMPILES(" +#include <fcntl.h> +#include <linux/falloc.h> +int main() { + int fd = open(\"/dev/null\", 0); + fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024); +} +" HAVE_FALLOCATE) + if(HAVE_FALLOCATE) + add_definitions(-DROCKSDB_FALLOCATE_PRESENT) + endif() +endif() + +CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE) +if(HAVE_MALLOC_USABLE_SIZE) + add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE) +endif() + +include_directories(${ROCKSDB_SOURCE_DIR}) +include_directories(${ROCKSDB_SOURCE_DIR}/include) +include_directories(SYSTEM ${ROCKSDB_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src) + +find_package(Threads REQUIRED) +if(WIN32) + set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib) +else() + set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} ${LIBRT}) +endif() + +set(ROCKSDB_LIBS rocksdblib}) +set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) + +#add_subdirectory(${ROCKSDB_SOURCE_DIR}/tools) + +# Main library source code + +set(ROCKSDB_SOURCES + cache/clock_cache.cc + cache/lru_cache.cc + cache/sharded_cache.cc + db/builder.cc + db/c.cc + db/column_family.cc + db/compacted_db_impl.cc + db/compaction.cc + db/compaction_iterator.cc + db/compaction_job.cc + db/compaction_picker.cc + db/compaction_picker_universal.cc + db/convenience.cc + db/db_filesnapshot.cc + db/db_impl.cc + db/db_impl_compaction_flush.cc + db/db_impl_debug.cc + db/db_impl_experimental.cc + db/db_impl_files.cc + db/db_impl_open.cc + db/db_impl_readonly.cc + db/db_impl_write.cc + db/db_info_dumper.cc + db/db_iter.cc + db/dbformat.cc ++ db/error_handler.cc + db/event_helpers.cc + db/experimental.cc + db/external_sst_file_ingestion_job.cc + db/file_indexer.cc + db/flush_job.cc + db/flush_scheduler.cc + db/forward_iterator.cc + db/internal_stats.cc + db/log_reader.cc + db/log_writer.cc ++ db/logs_with_prep_tracker.cc + db/malloc_stats.cc + db/managed_iterator.cc + db/memtable.cc + db/memtable_list.cc + db/merge_helper.cc + db/merge_operator.cc + db/range_del_aggregator.cc + db/repair.cc + db/snapshot_impl.cc + db/table_cache.cc + db/table_properties_collector.cc + db/transaction_log_impl.cc + db/version_builder.cc + db/version_edit.cc + db/version_set.cc + db/wal_manager.cc + db/write_batch.cc + db/write_batch_base.cc + db/write_controller.cc + db/write_thread.cc + env/env.cc + env/env_chroot.cc + env/env_hdfs.cc + env/mock_env.cc + memtable/alloc_tracker.cc + memtable/hash_cuckoo_rep.cc + memtable/hash_cuckoo_rep.cc + memtable/hash_linklist_rep.cc + memtable/hash_linklist_rep.cc + memtable/hash_skiplist_rep.cc + memtable/hash_skiplist_rep.cc + memtable/skiplistrep.cc + memtable/skiplistrep.cc + memtable/vectorrep.cc + memtable/vectorrep.cc + memtable/write_buffer_manager.cc + monitoring/histogram.cc + monitoring/histogram_windowing.cc + monitoring/instrumented_mutex.cc + monitoring/iostats_context.cc + monitoring/perf_context.cc + monitoring/perf_level.cc + monitoring/statistics.cc + monitoring/thread_status_impl.cc + monitoring/thread_status_updater.cc + monitoring/thread_status_util.cc + monitoring/thread_status_util_debug.cc + options/cf_options.cc + options/db_options.cc + options/options.cc + options/options_helper.cc + options/options_parser.cc + options/options_sanity_check.cc + port/stack_trace.cc + table/adaptive_table_factory.cc + table/block.cc + table/block_based_filter_block.cc + table/block_based_table_builder.cc + table/block_based_table_factory.cc + table/block_based_table_reader.cc + table/block_builder.cc + table/block_fetcher.cc + table/block_prefix_index.cc + table/bloom_block.cc + table/cuckoo_table_builder.cc + table/cuckoo_table_factory.cc + table/cuckoo_table_reader.cc + table/flush_block_policy.cc + table/format.cc + table/full_filter_block.cc + table/get_context.cc + table/index_builder.cc + table/iterator.cc + table/merging_iterator.cc + table/meta_blocks.cc + table/partitioned_filter_block.cc + table/persistent_cache_helper.cc + table/plain_table_builder.cc + table/plain_table_factory.cc + table/plain_table_index.cc + table/plain_table_key_coding.cc + table/plain_table_reader.cc + table/sst_file_writer.cc + table/table_properties.cc + table/two_level_iterator.cc + tools/db_bench_tool.cc + tools/dump/db_dump_tool.cc + tools/ldb_cmd.cc + tools/ldb_tool.cc + tools/sst_dump_tool.cc + util/arena.cc + util/auto_roll_logger.cc + util/bloom.cc + util/coding.cc + util/compaction_job_stats_impl.cc + util/comparator.cc ++ util/compression_context_cache.cc + util/concurrent_arena.cc + util/crc32c.cc + util/delete_scheduler.cc + util/dynamic_bloom.cc + util/event_logger.cc + util/file_reader_writer.cc + util/file_util.cc + util/filename.cc + util/filter_policy.cc + util/hash.cc + util/log_buffer.cc + util/murmurhash.cc + util/random.cc + util/rate_limiter.cc + util/slice.cc + util/sst_file_manager_impl.cc + util/status.cc + util/status_message.cc + util/string_util.cc + util/sync_point.cc ++ util/sync_point_impl.cc + util/testutil.cc + util/thread_local.cc + util/threadpool_imp.cc + util/transaction_test_util.cc + util/xxhash.cc + utilities/backupable/backupable_db.cc + utilities/blob_db/blob_db.cc + utilities/checkpoint/checkpoint_impl.cc + utilities/col_buf_decoder.cc + utilities/col_buf_encoder.cc + utilities/column_aware_encoding_util.cc + utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc + utilities/date_tiered/date_tiered_db_impl.cc + utilities/debug.cc + utilities/document/document_db.cc + utilities/document/json_document.cc + utilities/document/json_document_builder.cc + utilities/env_mirror.cc + utilities/geodb/geodb_impl.cc + utilities/leveldb_options/leveldb_options.cc + utilities/lua/rocks_lua_compaction_filter.cc + utilities/memory/memory_util.cc + utilities/merge_operators/max.cc + utilities/merge_operators/put.cc + utilities/merge_operators/string_append/stringappend.cc + utilities/merge_operators/string_append/stringappend2.cc + utilities/merge_operators/uint64add.cc + utilities/option_change_migration/option_change_migration.cc + utilities/options/options_util.cc + utilities/persistent_cache/block_cache_tier.cc + utilities/persistent_cache/block_cache_tier_file.cc + utilities/persistent_cache/block_cache_tier_metadata.cc + utilities/persistent_cache/persistent_cache_tier.cc + utilities/persistent_cache/volatile_tier_impl.cc + utilities/redis/redis_lists.cc + utilities/simulator_cache/sim_cache.cc + utilities/spatialdb/spatial_db.cc + utilities/table_properties_collectors/compact_on_deletion_collector.cc + utilities/transactions/optimistic_transaction_db_impl.cc + utilities/transactions/pessimistic_transaction.cc + utilities/transactions/pessimistic_transaction_db.cc + utilities/transactions/snapshot_checker.cc + utilities/transactions/transaction_base.cc + utilities/transactions/transaction_db_mutex_impl.cc + utilities/transactions/transaction_lock_mgr.cc + utilities/transactions/transaction_util.cc + utilities/transactions/write_prepared_txn.cc + utilities/transactions/write_prepared_txn_db.cc ++ utilities/transactions/write_unprepared_txn.cc ++ utilities/transactions/write_unprepared_txn_db.cc + utilities/ttl/db_ttl_impl.cc + utilities/write_batch_with_index/write_batch_with_index.cc + utilities/write_batch_with_index/write_batch_with_index_internal.cc +) + +if(WIN32) + list(APPEND ROCKSDB_SOURCES + port/win/io_win.cc + port/win/env_win.cc + port/win/env_default.cc + port/win/port_win.cc + port/win/win_logger.cc + port/win/win_thread.cc + port/win/xpress_win.cc) +else() + list(APPEND ROCKSDB_SOURCES + port/port_posix.cc + env/env_posix.cc + env/io_posix.cc) +endif() +SET(SOURCES) +FOREACH(s ${ROCKSDB_SOURCES}) + list(APPEND SOURCES ${ROCKSDB_SOURCE_DIR}/${s}) +ENDFOREACH() + +if(MSVC) + add_definitions(-DHAVE_SSE42 -DHAVE_PCLMUL) +else() + set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul ${CXX11_FLAGS}") + + CHECK_CXX_SOURCE_COMPILES(" +#include <cstdint> +#include <nmmintrin.h> +#include <wmmintrin.h> +int main() { + volatile uint32_t x = _mm_crc32_u32(0, 0); + const auto a = _mm_set_epi64x(0, 0); + const auto b = _mm_set_epi64x(0, 0); + const auto c = _mm_clmulepi64_si128(a, b, 0x00); + auto d = _mm_cvtsi128_si64(c); +} +" HAVE_SSE42) + if(HAVE_SSE42) + set_source_files_properties(${ROCKSDB_SOURCE_DIR}/util/crc32c.cc + PROPERTIES COMPILE_FLAGS "-DHAVE_SSE42 -DHAVE_PCLMUL -msse4.2 -mpclmul") + endif() + unset(CMAKE_REQUIRED_FLAGS) +endif() + +IF(CMAKE_VERSION VERSION_GREATER "2.8.10") + STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S") +ENDIF() + +CONFIGURE_FILE(${ROCKSDB_SOURCE_DIR}/util/build_version.cc.in build_version.cc @ONLY) +INCLUDE_DIRECTORIES(${ROCKSDB_SOURCE_DIR}/util) +list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/build_version.cc) + +ADD_CONVENIENCE_LIBRARY(rocksdblib ${SOURCES}) +target_link_libraries(rocksdblib ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) +IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set_target_properties(rocksdblib PROPERTIES COMPILE_FLAGS "-fPIC -fno-builtin-memcmp -frtti") +endif() + diff --cc storage/rocksdb/ha_rocksdb.cc index adef71e,c7d7cc0..61561a6 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@@ -127,6 -110,58 +128,60 @@@ const std::string DEFAULT_CF_NAME("defa const std::string DEFAULT_SYSTEM_CF_NAME("__system__"); const std::string PER_INDEX_CF_NAME("$per_index_cf"); + class Rdb_explicit_snapshot; + + std::mutex explicit_snapshot_mutex; + ulonglong explicit_snapshot_counter = 0; + std::unordered_map<ulonglong, std::weak_ptr<Rdb_explicit_snapshot>> + explicit_snapshots; + static std::vector<GL_INDEX_ID> rdb_indexes_to_recalc; + ++#ifdef MARIADB_NOT_YET + class Rdb_explicit_snapshot : public explicit_snapshot { + std::unique_ptr<rocksdb::ManagedSnapshot> snapshot; + + public: + static std::shared_ptr<Rdb_explicit_snapshot> + create(snapshot_info_st *ss_info, rocksdb::DB *db, + const rocksdb::Snapshot *snapshot) { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + auto s = std::unique_ptr<rocksdb::ManagedSnapshot>( + new rocksdb::ManagedSnapshot(db, snapshot)); + if (!s) { + return nullptr; + } + ss_info->snapshot_id = ++explicit_snapshot_counter; + auto ret = std::make_shared<Rdb_explicit_snapshot>(*ss_info, std::move(s)); + if (!ret) { + return nullptr; + } + explicit_snapshots[ss_info->snapshot_id] = ret; + return ret; + } + + static std::shared_ptr<Rdb_explicit_snapshot> + get(const ulonglong snapshot_id) { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + auto elem = explicit_snapshots.find(snapshot_id); + if (elem == explicit_snapshots.end()) { + return nullptr; + } + return elem->second.lock(); + } + + rocksdb::ManagedSnapshot *get_snapshot() { return snapshot.get(); } + + Rdb_explicit_snapshot(snapshot_info_st ss_info, + std::unique_ptr<rocksdb::ManagedSnapshot> snapshot) + : explicit_snapshot(ss_info), snapshot(std::move(snapshot)) {} + + virtual ~Rdb_explicit_snapshot() { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + explicit_snapshots.erase(ss_info.snapshot_id); + } + }; ++#endif + /** Updates row counters based on the table type and operation type. */ @@@ -247,8 -269,6 +284,10 @@@ struct Rdb_open_tables_map Rdb_table_handler *get_table_handler(const char *const table_name); void release_table_handler(Rdb_table_handler *const table_handler); + Rdb_open_tables_map() : m_hash(get_hash_key, system_charset_info) { } + ++ void free_hash(void) { m_hash.~Rdb_table_set(); } ++ std::vector<std::string> get_table_names(void) const; }; @@@ -506,77 -529,20 +549,85 @@@ static my_bool rocksdb_enable_bulk_load static my_bool rocksdb_print_snapshot_conflict_queries = 0; static my_bool rocksdb_large_prefix = 0; static my_bool rocksdb_allow_to_start_after_corruption = 0; +static char* rocksdb_git_hash; + +char *compression_types_val= + const_cast<char*>(get_rocksdb_supported_compression_types()); + static uint64_t rocksdb_write_policy = + rocksdb::TxnDBWritePolicy::WRITE_COMMITTED; + static my_bool rocksdb_error_on_suboptimal_collation = 1; + static uint32_t rocksdb_stats_recalc_rate = 0; + static uint32_t rocksdb_debug_manual_compaction_delay = 0; + static uint32_t rocksdb_max_manual_compactions = 0; std::atomic<uint64_t> rocksdb_row_lock_deadlocks(0); std::atomic<uint64_t> rocksdb_row_lock_wait_timeouts(0); std::atomic<uint64_t> rocksdb_snapshot_conflict_errors(0); std::atomic<uint64_t> rocksdb_wal_group_syncs(0); + std::atomic<uint64_t> rocksdb_manual_compactions_processed(0); + std::atomic<uint64_t> rocksdb_manual_compactions_running(0); + + +/* + Remove directory with files in it. + Used to remove checkpoint created by mariabackup. +*/ +#ifdef _WIN32 +#include <direct.h> /* unlink*/ +#ifndef F_OK +#define F_OK 0 +#endif +#endif + +static int rmdir_force(const char *dir) { + if (access(dir, F_OK)) + return true; + + char path[FN_REFLEN]; + char sep[] = {FN_LIBCHAR, 0}; + int err = 0; + + MY_DIR *dir_info = my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i = 0; i < dir_info->number_of_files; i++) { + FILEINFO *file = dir_info->dir_entry + i; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + err = my_delete(path, 0); + + if (err) { + break; + } + } + + my_dirend(dir_info); + + if (!err) + err = rmdir(dir); + + return (err == 0) ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; +} + + +static void rocksdb_remove_mariabackup_checkpoint( + my_core::THD *const, + struct st_mysql_sys_var *const , + void *const var_ptr, const void *const) { + std::string mariabackup_checkpoint_dir(rocksdb_datadir); + + mariabackup_checkpoint_dir.append("/mariabackup-checkpoint"); + + if (unlink(mariabackup_checkpoint_dir.c_str()) == 0) + return; + + rmdir_force(mariabackup_checkpoint_dir.c_str()); +} + + static std::unique_ptr<rocksdb::DBOptions> rdb_init_rocksdb_db_options(void) { auto o = std::unique_ptr<rocksdb::DBOptions>(new rocksdb::DBOptions()); @@@ -1077,9 -1076,11 +1169,11 @@@ static MYSQL_SYSVAR_INT(table_cache_num "DBOptions::table_cache_numshardbits for RocksDB", nullptr, nullptr, rocksdb_db_options->table_cache_numshardbits, - /* min */ 0, /* max */ INT_MAX, 0); + // LRUCache limits this to 19 bits, anything greater + // fails to create a cache and returns a nullptr + /* min */ 0, /* max */ 19, 0); -static MYSQL_SYSVAR_ULONG(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, +static MYSQL_SYSVAR_UINT64_T(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "DBOptions::WAL_ttl_seconds for RocksDB", nullptr, nullptr, rocksdb_db_options->WAL_ttl_seconds, @@@ -1753,7 -1772,11 +1877,12 @@@ static struct st_mysql_sys_var *rocksdb MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(allow_to_start_after_corruption), + MYSQL_SYSVAR(git_hash), + MYSQL_SYSVAR(error_on_suboptimal_collation), + MYSQL_SYSVAR(stats_recalc_rate), + MYSQL_SYSVAR(debug_manual_compaction_delay), + MYSQL_SYSVAR(max_manual_compactions), + MYSQL_SYSVAR(manual_compaction_threads), nullptr}; static rocksdb::WriteOptions @@@ -1897,9 -1941,7 +2068,9 @@@ private ulonglong m_writes_at_last_savepoint; protected: - THD *m_thd = nullptr; + - rocksdb::ReadOptions m_read_opts; ++protected: + THD *m_thd = nullptr; static std::multiset<Rdb_transaction *> s_tx_list; static mysql_mutex_t s_tx_list_mutex; @@@ -1956,14 -1990,20 +2127,15 @@@ protected return s; } -protected: - /* - The following two are helper functions to be overloaded by child classes. - They should provide RocksDB's savepoint semantics. - */ - virtual void do_set_savepoint() = 0; - virtual void do_rollback_to_savepoint() = 0; - public: + rocksdb::ReadOptions m_read_opts; const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; +#ifdef MARIAROCKS_NOT_YET + // TODO: MariaDB probably doesn't need these at all: const char *m_mysql_gtid; const char *m_mysql_max_gtid; +#endif String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; @@@ -2666,7 -2697,15 +2841,18 @@@ public void acquire_snapshot(bool acquire_now) override { if (m_read_opts.snapshot == nullptr) { ++#ifdef MARIAROCKS_NOT_YET + const auto thd_ss = std::static_pointer_cast<Rdb_explicit_snapshot>( + m_thd->get_explicit_snapshot()); + if (thd_ss) { + m_explicit_snapshot = thd_ss; + } + if (m_explicit_snapshot) { + auto snapshot = m_explicit_snapshot->get_snapshot()->snapshot(); + snapshot_created(snapshot); - } else if (is_tx_read_only()) { ++ } else ++#endif + if (is_tx_read_only()) { snapshot_created(rdb->GetSnapshot()); } else if (acquire_now) { m_rocksdb_tx->SetSnapshot(); @@@ -2683,7 -2722,10 +2869,13 @@@ if (m_read_opts.snapshot != nullptr) { m_snapshot_timestamp = 0; ++#ifdef MARIAROCKS_NOT_YET + if (m_explicit_snapshot) { + m_explicit_snapshot.reset(); + need_clear = false; - } else if (is_tx_read_only()) { ++ } else ++#endif + if (is_tx_read_only()) { rdb->ReleaseSnapshot(m_read_opts.snapshot); need_clear = false; } else { @@@ -2946,6 -2991,6 +3143,7 @@@ error m_batch->RollbackToSavePoint(); } ++ public: bool is_writebatch_trx() const override { return true; } @@@ -3563,7 -3494,8 +3768,8 @@@ static int rocksdb_commit(handlerton* h /* We get here when committing a statement within a transaction. */ - tx->set_tx_failed(false); + tx->make_stmt_savepoint_permanent(); + tx->make_stmt_savepoint_permanent(); } if (my_core::thd_tx_isolation(thd) <= ISO_READ_COMMITTED) { @@@ -4066,23 -3998,24 +4275,23 @@@ static bool rocksdb_show_status(handler str.clear(); rocksdb::MemoryUtil::GetApproximateMemoryUsageByType(dbs, cache_set, &temp_usage_by_type); - - snprintf(buf, sizeof(buf), "\nMemTable Total: %lu", - temp_usage_by_type[rocksdb::MemoryUtil::kMemTableTotal]); + snprintf(buf, sizeof(buf), "\nMemTable Total: %llu", + (ulonglong)temp_usage_by_type[rocksdb::MemoryUtil::kMemTableTotal]); str.append(buf); - snprintf(buf, sizeof(buf), "\nMemTable Unflushed: %lu", - temp_usage_by_type[rocksdb::MemoryUtil::kMemTableUnFlushed]); + snprintf(buf, sizeof(buf), "\nMemTable Unflushed: %llu", + (ulonglong)temp_usage_by_type[rocksdb::MemoryUtil::kMemTableUnFlushed]); str.append(buf); - snprintf(buf, sizeof(buf), "\nTable Readers Total: %lu", - temp_usage_by_type[rocksdb::MemoryUtil::kTableReadersTotal]); + snprintf(buf, sizeof(buf), "\nTable Readers Total: %llu", + (ulonglong)temp_usage_by_type[rocksdb::MemoryUtil::kTableReadersTotal]); str.append(buf); - snprintf(buf, sizeof(buf), "\nCache Total: %lu", - temp_usage_by_type[rocksdb::MemoryUtil::kCacheTotal]); + snprintf(buf, sizeof(buf), "\nCache Total: %llu", + (ulonglong)temp_usage_by_type[rocksdb::MemoryUtil::kCacheTotal]); str.append(buf); - snprintf(buf, sizeof(buf), "\nDefault Cache Capacity: %lu", - internal_cache_count * kDefaultInternalCacheSize); + snprintf(buf, sizeof(buf), "\nDefault Cache Capacity: %llu", + (ulonglong)internal_cache_count * kDefaultInternalCacheSize); str.append(buf); res |= print_stats(thd, "MEMORY_STATS", "rocksdb", str, stat_print); - #ifdef MARIAROCKS_NOT_YET + /* Show the background thread status */ std::vector<rocksdb::ThreadStatus> thread_list; rocksdb::Status s = rdb->GetEnv()->GetThreadList(&thread_list); @@@ -4119,9 -4052,24 +4328,28 @@@ str, stat_print); } } + ++#ifdef MARIAROCKS_NOT_YET + /* Explicit snapshot information */ + str.clear(); + { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + for (const auto &elem : explicit_snapshots) { + const auto &ss = elem.second.lock(); + DBUG_ASSERT(ss != nullptr); + const auto &info = ss->ss_info; + str += "\nSnapshot ID: " + std::to_string(info.snapshot_id) + + "\nBinlog File: " + info.binlog_file + + "\nBinlog Pos: " + std::to_string(info.binlog_pos) + + "\nGtid Executed: " + info.gtid_executed + "\n"; + } + } +#endif + + if (!str.empty()) { + res |= print_stats(thd, "EXPLICIT_SNAPSHOTS", "rocksdb", str, stat_print); + } +#ifdef MARIAROCKS_NOT_YET } else if (stat_type == HA_ENGINE_TRX) { /* Handle the SHOW ENGINE ROCKSDB TRANSACTION STATUS command */ res |= rocksdb_show_snapshot_status(hton, thd, stat_print); @@@ -4141,8 -4089,48 +4369,52 @@@ static inline void rocksdb_register_tx( } } +static const char *ha_rocksdb_exts[] = {NullS}; + ++#ifdef MARIAROCKS_NOT_YET + static bool rocksdb_explicit_snapshot( + handlerton *const /* hton */, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle */ + snapshot_info_st *ss_info) /*!< out: Snapshot information */ + { + switch (ss_info->op) { + case snapshot_operation::SNAPSHOT_CREATE: { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + auto s = Rdb_explicit_snapshot::create(ss_info, rdb, rdb->GetSnapshot()); + if (mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + thd->set_explicit_snapshot(s); + return s == nullptr; + } + case snapshot_operation::SNAPSHOT_ATTACH: { + auto s = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!s) { + return true; + } + *ss_info = s->ss_info; + thd->set_explicit_snapshot(s); + return false; + } + case snapshot_operation::SNAPSHOT_RELEASE: { + if (!thd->get_explicit_snapshot()) { + return true; + } + *ss_info = thd->get_explicit_snapshot()->ss_info; + thd->set_explicit_snapshot(nullptr); + return false; + } + default: + DBUG_ASSERT(false); + return true; + } + return true; + } ++#endif + /* Supporting START TRANSACTION WITH CONSISTENT [ROCKSDB] SNAPSHOT @@@ -4165,10 -4153,12 +4437,15 @@@ InnoDB and RocksDB transactions. */ static int rocksdb_start_tx_and_assign_read_view( - handlerton *const hton, /*!< in: RocksDB handlerton */ - THD* thd) /*!< in: MySQL thread handle of the - user for whom the transaction should - be committed */ + handlerton *const hton, /*!< in: RocksDB handlerton */ - THD *const thd, /*!< in: MySQL thread handle of the ++ THD *const thd /*!< in: MySQL thread handle of the + user for whom the transaction should + be committed */ ++) ++#ifdef MARIAROCKS_NOT_YET + snapshot_info_st *ss_info) /*!< in/out: Snapshot info like binlog file, pos, + gtid executed and snapshot ID */ ++#endif { ulong const tx_isolation = my_core::thd_tx_isolation(thd); @@@ -4176,16 -4166,14 +4453,27 @@@ my_error(ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT, MYF(0)); return HA_EXIT_FAILURE; } + ++#ifdef MARIADB_NOT_YET + if (ss_info) { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } else { + return HA_EXIT_FAILURE; + } - } ++#endif ++ + /* + MariaDB: there is no need to call mysql_bin_log_lock_commits and then + unlock back. + SQL layer calls start_consistent_snapshot() for all engines, including the + binlog under LOCK_commit_ordered mutex. ++ + The mutex prevents binlog commits from happening (right?) while the storage + engine(s) allocate read snapshots. That way, each storage engine is - synchronized with current binlog position. ++ synchronized with current binlog position. + */ + mysql_mutex_assert_owner(&LOCK_commit_ordered); Rdb_transaction *const tx = get_or_create_tx(thd); Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); @@@ -4195,9 -4183,94 +4483,106 @@@ rocksdb_register_tx(hton, thd, tx); tx->acquire_snapshot(true); ++#ifdef MARIADB_NOT_YET + if (ss_info) { + mysql_bin_log_unlock_commits(ss_info); + } - ++#endif return HA_EXIT_SUCCESS; } ++#ifdef MARIADB_NOT_YET + static int rocksdb_start_tx_with_shared_read_view( + handlerton *const hton, /*!< in: RocksDB handlerton */ - THD *const thd, /*!< in: MySQL thread handle of the ++ THD *const thd) /*!< in: MySQL thread handle of the + user for whom the transaction should + be committed */ ++#ifdef MARIADB_NOT_YET + snapshot_info_st *ss_info) /*!< out: Snapshot info like binlog file, pos, + gtid executed and snapshot ID */ ++#endif + { + DBUG_ASSERT(thd != nullptr); - DBUG_ASSERT(ss_info != nullptr); + + int error = HA_EXIT_SUCCESS; + + ulong const tx_isolation = my_core::thd_tx_isolation(thd); + if (tx_isolation != ISO_REPEATABLE_READ) { + my_error(ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT, MYF(0)); + return HA_EXIT_FAILURE; + } + ++ Rdb_transaction *tx = nullptr; ++#ifdef MARIADB_NOT_YET + std::shared_ptr<Rdb_explicit_snapshot> explicit_snapshot; + const auto op = ss_info->op; - Rdb_transaction *tx = nullptr; + + DBUG_ASSERT(op == snapshot_operation::SNAPSHOT_CREATE || + op == snapshot_operation::SNAPSHOT_ATTACH); + + // case: if binlogs are available get binlog file/pos and gtid info + if (op == snapshot_operation::SNAPSHOT_CREATE && mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + + if (op == snapshot_operation::SNAPSHOT_ATTACH) { + explicit_snapshot = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!explicit_snapshot) { + my_printf_error(ER_UNKNOWN_ERROR, "Snapshot %llu does not exist", MYF(0), + ss_info->snapshot_id); + error = HA_EXIT_FAILURE; + } + } ++#endif + + // case: all good till now + if (error == HA_EXIT_SUCCESS) { + tx = get_or_create_tx(thd); + Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); + ++#ifdef MARIADB_NOT_YET + if (explicit_snapshot) { + tx->m_explicit_snapshot = explicit_snapshot; + } ++#endif + + DBUG_ASSERT(!tx->has_snapshot()); + tx->set_tx_read_only(true); + rocksdb_register_tx(hton, thd, tx); + tx->acquire_snapshot(true); + ++#ifdef MARIADB_NOT_YET + // case: an explicit snapshot was not assigned to this transaction + if (!tx->m_explicit_snapshot) { + tx->m_explicit_snapshot = + Rdb_explicit_snapshot::create(ss_info, rdb, tx->m_read_opts.snapshot); + if (!tx->m_explicit_snapshot) { + my_printf_error(ER_UNKNOWN_ERROR, "Could not create snapshot", MYF(0)); + error = HA_EXIT_FAILURE; + } + } ++#endif + } + ++#ifdef MARIADB_NOT_YET + // case: unlock the binlog + if (op == snapshot_operation::SNAPSHOT_CREATE && mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + DBUG_ASSERT(error == HA_EXIT_FAILURE || tx->m_explicit_snapshot); + + // copy over the snapshot details to pass to the upper layers + if (tx->m_explicit_snapshot) { + *ss_info = tx->m_explicit_snapshot->ss_info; + ss_info->op = op; + } ++#endif + + return error; + } ++#endif + /* Dummy SAVEPOINT support. This is needed for long running transactions * like mysqldump (https://bugs.mysql.com/bug.php?id=71017) * Current SAVEPOINT does not correctly handle ROLLBACK and does not return @@@ -4445,6 -4502,9 +4832,8 @@@ static int rocksdb_init_func(void *cons mysql_mutex_init(rdb_sysvars_psi_mutex_key, &rdb_sysvars_mutex, MY_MUTEX_INIT_FAST); + mysql_mutex_init(rdb_block_cache_resize_mutex_key, + &rdb_block_cache_resize_mutex, MY_MUTEX_INIT_FAST); - rdb_open_tables.init_hash(); Rdb_transaction::init_mutex(); rocksdb_hton->state = SHOW_OPTION_YES; @@@ -4457,16 -4514,14 +4846,22 @@@ rocksdb_hton->commit_by_xid = rocksdb_commit_by_xid; rocksdb_hton->rollback_by_xid = rocksdb_rollback_by_xid; rocksdb_hton->recover = rocksdb_recover; + + rocksdb_hton->commit_ordered= rocksdb_commit_ordered; rocksdb_hton->commit = rocksdb_commit; + + rocksdb_hton->commit_checkpoint_request= rocksdb_checkpoint_request; + rocksdb_hton->rollback = rocksdb_rollback; - rocksdb_hton->db_type = DB_TYPE_ROCKSDB; rocksdb_hton->show_status = rocksdb_show_status; ++#ifdef MARIADB_NOT_YET + rocksdb_hton->explicit_snapshot = rocksdb_explicit_snapshot; ++#endif rocksdb_hton->start_consistent_snapshot = rocksdb_start_tx_and_assign_read_view; ++#ifdef MARIADB_NOT_YET + rocksdb_hton->start_shared_snapshot = rocksdb_start_tx_with_shared_read_view; ++#endif rocksdb_hton->savepoint_set = rocksdb_savepoint; rocksdb_hton->savepoint_rollback = rocksdb_rollback_to_savepoint; rocksdb_hton->savepoint_rollback_can_release_mdl = @@@ -4710,9 -4791,10 +5136,9 @@@ } cf_manager.init(std::move(cf_options_map), &cf_handles); - if (dict_manager.init(rdb->GetBaseDB(), &cf_manager)) { + if (dict_manager.init(rdb, &cf_manager)) { // NO_LINT_DEBUG sql_print_error("RocksDB: Failed to initialize data dictionary."); - rdb_open_tables.free_hash(); DBUG_RETURN(HA_EXIT_FAILURE); } @@@ -4877,66 -4968,28 +5318,77 @@@ static int rocksdb_done_func(void *cons sql_print_error("RocksDB: Couldn't stop the index thread: (errno=%d)", err); } + // signal the manual compaction thread to stop + rdb_mc_thread.signal(true); + // Wait for the manual compaction thread to finish. + err = rdb_mc_thread.join(); + if (err != 0) { + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Couldn't stop the manual compaction thread: (errno=%d)", err); + } + - if (rdb_open_tables.m_hash.records) { + if (rdb_open_tables.m_hash.size()) { // Looks like we are getting unloaded and yet we have some open tables // left behind. error = 1; } - rdb_open_tables.free_hash(); + /* + MariaDB: When the plugin is unloaded with UNINSTALL SONAME command, some + connections may still have Rdb_transaction objects. + + These objects are not genuine transactions (as SQL layer makes sure that + a plugin that is being unloaded has no open tables), they are empty + Rdb_transaction objects that were left there to save on object + creation/deletion. + + Go through the list and delete them. + */ + { + class Rdb_trx_deleter: public Rdb_tx_list_walker { + public: + std::set<Rdb_transaction*> rdb_trxs; + + void process_tran(const Rdb_transaction *const tx) override { + /* + Check if the transaction is really empty. We only check + non-WriteBatch-based transactions, because there is no easy way to + check WriteBatch-based transactions. + */ + if (!tx->is_writebatch_trx()) { + const auto tx_impl = static_cast<const Rdb_transaction_impl *>(tx); + DBUG_ASSERT(tx_impl); + if (tx_impl->get_rdb_trx()) + DBUG_ASSERT(0); + } + rdb_trxs.insert((Rdb_transaction*)tx); + }; + } deleter; + + Rdb_transaction::walk_tx_list(&deleter); + + for (std::set<Rdb_transaction*>::iterator it= deleter.rdb_trxs.begin(); + it != deleter.rdb_trxs.end(); + ++it) + { + // When a transaction is deleted, it removes itself from s_tx_list. + delete *it; + } + } + + /* + destructors for static objects can be called at _exit(), + but we want to free the memory at dlclose() + */ + rdb_open_tables.m_hash.~Rdb_table_set(); mysql_mutex_destroy(&rdb_open_tables.m_mutex); mysql_mutex_destroy(&rdb_sysvars_mutex); + mysql_mutex_destroy(&rdb_block_cache_resize_mutex); + delete rdb_collation_exceptions; + mysql_mutex_destroy(&rdb_collation_data_mutex); mysql_mutex_destroy(&rdb_mem_cmp_space_mutex); @@@ -6307,14 -6348,13 +6754,14 @@@ void ha_rocksdb::free_key_buffers() my_free(m_dup_sk_packed_tuple_old); m_dup_sk_packed_tuple_old = nullptr; - my_free(m_eq_cond_upper_bound); - m_eq_cond_upper_bound = nullptr; + my_free(m_scan_it_lower_bound); + m_scan_it_lower_bound = nullptr; - my_free(m_eq_cond_lower_bound); - m_eq_cond_lower_bound = nullptr; + my_free(m_scan_it_upper_bound); + m_scan_it_upper_bound = nullptr; } +#ifdef MARIAROCKS_NOT_YET void ha_rocksdb::set_use_read_free_rpl(const char *const whitelist) { DBUG_ASSERT(whitelist != nullptr); @@@ -6796,12 -6766,12 +7241,10 @@@ int ha_rocksdb::create_cfs std::array<struct key_def_cf_info, MAX_INDEXES + 1> *const cfs) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); char tablename_sys[NAME_LEN + 1]; - - my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), - tablename_sys, sizeof(tablename_sys)); + bool tsys_set= false; /* The first loop checks the index parameters and creates @@@ -7212,10 -7163,7 +7639,8 @@@ int ha_rocksdb::create_key_def(const TA int rdb_normalize_tablename(const std::string &tablename, std::string *const strbuf) { - DBUG_ASSERT(strbuf != nullptr); - - if (tablename.size() < 2 || tablename[0] != '.' || tablename[1] != '/') { + if (tablename.size() < 2 || tablename[0] != '.' || + (tablename[1] != FN_LIBCHAR && tablename[1] != FN_LIBCHAR2)) { DBUG_ASSERT(0); // We were not passed table name? return HA_ERR_ROCKSDB_INVALID_TABLE; } @@@ -7875,11 -7809,10 +8294,10 @@@ ulong ha_rocksdb::index_flags(uint inx pair for. */ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { - DBUG_ASSERT(buf != nullptr); DBUG_ASSERT(table != nullptr); - +#ifdef MARIAROCKS_NOT_YET stats.rows_requested++; - +#endif /* Use STATUS_NOT_FOUND when record not found or some error occurred */ table->status = STATUS_NOT_FOUND; @@@ -9798,34 -9666,23 +10203,28 @@@ int ha_rocksdb::update_write_row(const If the index was reverse order, upper bound would be 0x0000b3eb003f65c5e78857, and lower bound would be 0x0000b3eb003f65c5e78859. These cover given eq condition range. + + @param lower_bound_buf IN Buffer for lower bound + @param upper_bound_buf IN Buffer for upper bound + + @param outer_u */ - void ha_rocksdb::setup_iterator_bounds(const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { - uint eq_cond_len = eq_cond.size(); - memcpy(upper_bound_buf, eq_cond.data(), eq_cond_len); - kd.successor(upper_bound_buf, eq_cond_len); - memcpy(lower_bound_buf, eq_cond.data(), eq_cond_len); - kd.predecessor(lower_bound_buf, eq_cond_len); + void ha_rocksdb::setup_iterator_bounds( + const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, size_t bound_len, + uchar *const lower_bound, uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, rocksdb::Slice *upper_bound_slice) { + uint min_len = std::min(eq_cond.size(), bound_len); + memcpy(upper_bound, eq_cond.data(), min_len); + kd.successor(upper_bound, min_len); + memcpy(lower_bound, eq_cond.data(), min_len); + kd.predecessor(lower_bound, min_len); if (kd.m_is_reverse_cf) { - *out_upper_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); } else { - *out_upper_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); } } @@@ -9955,9 -9810,10 +10352,9 @@@ int ha_rocksdb::rnd_next(uchar *const b DBUG_ENTER_FUNC(); int rc; - ha_statistic_increment(&SSV::ha_read_rnd_next_count); for (;;) { rc = rnd_next_with_direction(buf, true); - if (rc != HA_ERR_LOCK_DEADLOCK || !m_rnd_scan_is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !m_rnd_scan_is_new_snapshot) break; /* exit the loop */ // release the snapshot and iterator and then regenerate them @@@ -10362,12 -10214,23 +10759,23 @@@ int ha_rocksdb::info(uint flag) continue; } KEY *const k = &table->key_info[i]; - for (uint j = 0; j < k->actual_key_parts; j++) { + for (uint j = 0; j < k->ext_key_parts; j++) { const Rdb_index_stats &k_stats = m_key_descr_arr[i]->m_stats; - uint x = k_stats.m_distinct_keys_per_prefix.size() > j && - k_stats.m_distinct_keys_per_prefix[j] > 0 - ? k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j] - : 0; + uint x; + + if (k_stats.m_distinct_keys_per_prefix.size() > j && + k_stats.m_distinct_keys_per_prefix[j] > 0) { + x = k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j]; + /* + If the number of rows is less than the number of prefixes (due to + sampling), the average number of rows with the same prefix is 1. + */ + if (x == 0) { + x = 1; + } + } else { + x = 0; + } if (x > stats.records) x = stats.records; if ((x == 0 && rocksdb_debug_optimizer_no_zero_cardinality) || @@@ -10736,6 -10548,11 +11144,13 @@@ int ha_rocksdb::external_lock(THD *cons DBUG_RETURN(HA_ERR_UNSUPPORTED); } ++#ifdef MARIADB_NOT_YET + if (thd->get_explicit_snapshot()) { + my_error(ER_UPDATES_WITH_EXPLICIT_SNAPSHOT, MYF(0)); + DBUG_RETURN(HA_ERR_UNSUPPORTED); + } ++#endif + /* SQL layer signals us to take a write lock. It does so when starting DML statement. We should put locks on the rows we're reading. @@@ -12807,6 -12661,46 +13242,42 @@@ void Rdb_background_thread::run() rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); } } - + // Recalculate statistics for indexes. + if (rocksdb_stats_recalc_rate) { + std::unordered_map<GL_INDEX_ID, std::shared_ptr<const Rdb_key_def>> + to_recalc; + + if (rdb_indexes_to_recalc.empty()) { + struct Rdb_index_collector : public Rdb_tables_scanner { + int add_table(Rdb_tbl_def *tdef) override { + for (uint i = 0; i < tdef->m_key_count; i++) { + rdb_indexes_to_recalc.push_back( + tdef->m_key_descr_arr[i]->get_gl_index_id()); + } + return HA_EXIT_SUCCESS; + } + } collector; + ddl_manager.scan_for_tables(&collector); + } + + while (to_recalc.size() < rocksdb_stats_recalc_rate && + !rdb_indexes_to_recalc.empty()) { + const auto index_id = rdb_indexes_to_recalc.back(); + rdb_indexes_to_recalc.pop_back(); + + std::shared_ptr<const Rdb_key_def> keydef = + ddl_manager.safe_find(index_id); + + if (keydef) { + to_recalc.insert(std::make_pair(keydef->get_gl_index_id(), keydef)); + } + } + + if (!to_recalc.empty()) { + calculate_stats(to_recalc, false); + } + } + - // Set the next timestamp for mysql_cond_timedwait() (which ends up calling - // pthread_cond_timedwait()) to wait on. - ts_next_sync.tv_sec = ts.tv_sec + WAKE_UP_INTERVAL; } // save remaining stats which might've left unsaved @@@ -12973,11 -13000,6 +13577,8 @@@ const char *get_rdb_io_error_string(con // In case of core dump generation we want this function NOT to be optimized // so that we can capture as much data as possible to debug the root cause // more efficiently. +#ifdef __GNUC__ - #pragma GCC push_options - #pragma GCC optimize("O0") +#endif - void rdb_handle_io_error(const rocksdb::Status status, const RDB_IO_ERROR_TYPE err_type) { if (status.IsIOError()) { @@@ -13026,10 -13051,7 +13630,8 @@@ } } } - +#ifdef __GNUC__ - #pragma GCC pop_options +#endif - Rdb_dict_manager *rdb_get_dict_manager(void) { return &dict_manager; } Rdb_ddl_manager *rdb_get_ddl_manager(void) { return &ddl_manager; } @@@ -13573,7 -13605,6 +14218,7 @@@ maria_declare_plugin(rocksdb_se) myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, myrocks::rdb_i_s_cfoptions, myrocks::rdb_i_s_compact_stats, myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, - myrocks::rdb_i_s_index_file_map, myrocks::rdb_i_s_lock_info, - myrocks::rdb_i_s_trx_info, + myrocks::rdb_i_s_sst_props, myrocks::rdb_i_s_index_file_map, + myrocks::rdb_i_s_lock_info, myrocks::rdb_i_s_trx_info, - myrocks::rdb_i_s_deadlock_info mysql_declare_plugin_end; + myrocks::rdb_i_s_deadlock_info +maria_declare_plugin_end; diff --cc storage/rocksdb/ha_rocksdb.h index 739a2c8,ca14165..c444710 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@@ -1402,8 -1369,7 +1407,8 @@@ public virtual void rpl_after_delete_rows() override; virtual void rpl_before_update_rows() override; virtual void rpl_after_update_rows() override; - virtual bool use_read_free_rpl(); + virtual bool use_read_free_rpl() override; +#endif // MARIAROCKS_NOT_YET private: /* Flags tracking if we are inside different replication operation */ diff --cc storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result index 08f2329,9731007..0617232 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result @@@ -6,18 -6,16 +6,18 @@@ LOAD DATA INFILE <input_file> INTO TABL set rocksdb_bulk_load=0; select count(pk) from t1; count(pk) -3000000 +300000 select count(a) from t1; count(a) -3000000 +300000 select count(b) from t1; count(b) -3000000 +300000 ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; - ERROR HY000: Status error 10 received from RocksDB: Operation aborted: Failed to acquire lock due to max_num_locks limit + ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to max_num_locks limit' from ROCKSDB set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=0; diff --cc storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result index a8d5c07,6f811e1..de8a1fb --- a/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result @@@ -276,16 -276,12 +276,16 @@@ 10 20 30 - connect con1,localhost,root,,; - connection con1; alter table t1 force; - connection default; + select * from t1; + a + insert into t1 values (100); select * from t1; a +connection con1; +insert into t1 values (100); +select * from t1; +a 1 2 100 diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result index fe08cd7,8b51c12..604e557 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result @@@ -56,52 -56,44 +56,44 @@@ insert into t values () begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_before"; +set debug_dbug="+d,crash_commit_before"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After engine prepare begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_prepare"; +set debug_dbug="+d,crash_commit_after_prepare"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After binlog begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_log"; +set debug_dbug="+d,crash_commit_after_log"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 6 - select max(i) from t; - max(i) - 5 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After everything begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after"; +set debug_dbug="+d,crash_commit_after"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 8 - select max(i) from t; - max(i) - 7 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 0c49622,17e6bed..f59b841 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@@ -150,11 -151,11 +150,29 @@@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +#--------------------------------------------------------------- +# MDEV-16703 Assertion failed in load_auto_incr_value_from_index +#--------------------------------------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++#---------------------------------- ++# Issue #792 Crash in autoincrement ++#---------------------------------- ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; + #---------------------------------- + # Issue #869 Crash in autoincrement + #---------------------------------- + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/collation.result index e372cbe,b4354ad..10e0d9b --- a/storage/rocksdb/mysql-test/rocksdb/r/collation.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/collation.result @@@ -1,11 -1,12 +1,12 @@@ - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - DROP TABLE IF EXISTS t1; + call mtr.add_suppression("Invalid pattern"); CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). DROP TABLE t1; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (latin1_bin, binary, utf8_bin). CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (latin1_bin, binary, utf8_bin). SET GLOBAL rocksdb_strict_collation_check=0; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; DROP TABLE t1; @@@ -124,7 -125,19 +126,19 @@@ ERROR HY000: Unsupported collation on s SET GLOBAL rocksdb_strict_collation_exceptions="abc"; CREATE TABLE abc (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; CREATE TABLE abcd (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (latin1_bin, binary, utf8_bin). DROP TABLE abc; SET GLOBAL rocksdb_strict_collation_exceptions=null; - SET GLOBAL rocksdb_strict_collation_exceptions=@start_global_value; + SET GLOBAL rocksdb_strict_collation_check=1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset latin1 collate latin1_bin; + ALTER TABLE t1 collate=latin1_general_ci; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result index 36db920,2a00fcb..0619e2d --- a/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result @@@ -13,8 -9,9 +13,9 @@@ connection default show create table information_schema.rocksdb_deadlock; Table Create Table ROCKSDB_DEADLOCK CREATE TEMPORARY TABLE `ROCKSDB_DEADLOCK` ( - `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT '0', + `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT 0, + `TIMESTAMP` bigint(8) NOT NULL DEFAULT '0', - `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT '0', + `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT 0, `CF_NAME` varchar(193) NOT NULL DEFAULT '', `WAITING_KEY` varchar(513) NOT NULL DEFAULT '', `LOCK_TYPE` varchar(193) NOT NULL DEFAULT '', @@@ -25,9 -22,8 +26,9 @@@ create table t (i int primary key) engine=rocksdb; insert into t values (1), (2), (3); select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #1 +connection con1; begin; select * from t where i=1 for update; i @@@ -47,13 -39,11 +48,13 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 Deadlock #2 +connection con1; begin; select * from t where i=1 for update; i @@@ -73,16 -59,14 +74,16 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 10; Deadlock #3 +connection con1; begin; select * from t where i=1 for update; i @@@ -102,24 -82,21 +103,24 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 1; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 +connection con3; set rocksdb_deadlock_detect_depth = 2; Deadlock #4 +connection con1; begin; select * from t where i=1 for update; i @@@ -150,12 -120,10 +151,12 @@@ connection con1 i 2 rollback; +connection default; set global rocksdb_max_latest_deadlocks = 5; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #5 +connection con1; begin; select * from t where i=1 for update; i @@@ -191,16 -154,11 +192,16 @@@ connection con1 i 2 rollback; +connection con3; rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 +disconnect con1; +disconnect con2; +disconnect con3; set global rocksdb_lock_wait_timeout = @prior_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_deadlock_detect; drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result index 4370715,80d95e4..e024873 --- a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result @@@ -266,8 -266,8 +266,8 @@@ explai select * from t0,t1 where (t0.key1=t1.key1) and (t0.key1=3 or t0.key2<4) and t1.key1=2; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t0 ref i1,i2 i1 4 const 1 Using where + 1 SIMPLE t0 ref i1,i2 i1 4 const 2 Using where -1 SIMPLE t1 ref i1 i1 4 const 2 NULL +1 SIMPLE t1 ref i1 i1 4 const 1 explain select * from t0,t1 where t0.key1 = 5 and (t1.key1 = t0.key1 or t1.key8 = t0.key1); id select_type table type possible_keys key key_len ref rows Extra @@@ -281,12 -281,13 +281,12 @@@ id select_type table type possible_key explain select * from t1 where key1=3 or key2=4 union select * from t1 where key1<4 or key3=5; id select_type table type possible_keys key key_len ref rows Extra - 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where - 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 2 Using sort_union(i1,i3); Using where + 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where + 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 4 Using sort_union(i1,i3); Using where -NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL Using temporary +NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL explain select * from (select * from t1 where key1 = 3 or key2 =3) as Z where key8 >5; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY <derived2> ALL NULL NULL NULL NULL 4 Using where -2 DERIVED t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where +1 SIMPLE t1 index_merge i1,i2,i8 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where create table t3 like t0; insert into t3 select * from t0; alter table t3 add key9 int not null, add index i9(key9); @@@ -1321,16 -1324,16 +1321,16 @@@ id select_type table type possible_key 1 SIMPLE t1 ref key1 key1 4 const ROWS Using where explain select * from t1 where pk1 < 7500 and key1 = 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 PRIMARY 4 NULL ROWS Using where +1 SIMPLE t1 range PRIMARY,key1 key1 8 NULL ROWS Using index condition explain select * from t1 where pktail1ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail1ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail1ok key1,pktail1ok 4,4 NULL 1 Using intersect(key1,pktail1ok); Using where explain select * from t1 where pktail2ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail2ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail2ok key1,pktail2ok 4,4 NULL 1 Using intersect(key1,pktail2ok); Using where explain select * from t1 where (pktail2ok=1 and pk1< 50000) or key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok pktail2ok,key1 8,4 NULL ROWS Using sort_union(pktail2ok,key1); Using where + 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok PRIMARY,key1 4,4 NULL ROWS Using union(PRIMARY,key1); Using where explain select * from t1 where pktail3bad=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref key1,pktail3bad EITHER_KEY 4 const ROWS Using where diff --cc storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index aba14e3,f830440..e22a85f --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@@ -16,8 -12,12 +16,9 @@@ SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WOR select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index dc13bb3,f9a3f81..c6e15dc --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@@ -915,7 -904,8 +918,8 @@@ rocksdb_enable_ttl O rocksdb_enable_ttl_read_filtering ON rocksdb_enable_write_thread_adaptive_yield OFF rocksdb_error_if_exists OFF + rocksdb_error_on_suboptimal_collation ON -rocksdb_flush_log_at_trx_commit 1 +rocksdb_flush_log_at_trx_commit 0 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF @@@ -1457,111 -1446,117 +1465,115 @@@ drop table t0, t1 # show status like 'rocksdb%'; Variable_name Value -rocksdb_rows_deleted # -rocksdb_rows_inserted # -rocksdb_rows_read # -rocksdb_rows_updated # -rocksdb_rows_deleted_blind # -rocksdb_rows_expired # -rocksdb_rows_filtered # -rocksdb_system_rows_deleted # -rocksdb_system_rows_inserted # -rocksdb_system_rows_read # -rocksdb_system_rows_updated # -rocksdb_memtable_total # -rocksdb_memtable_unflushed # -rocksdb_queries_point # -rocksdb_queries_range # -rocksdb_covered_secondary_key_lookups # -rocksdb_block_cache_add # -rocksdb_block_cache_add_failures # -rocksdb_block_cache_bytes_read # -rocksdb_block_cache_bytes_write # -rocksdb_block_cache_data_add # -rocksdb_block_cache_data_bytes_insert # -rocksdb_block_cache_data_hit # -rocksdb_block_cache_data_miss # -rocksdb_block_cache_filter_add # -rocksdb_block_cache_filter_bytes_evict # -rocksdb_block_cache_filter_bytes_insert # -rocksdb_block_cache_filter_hit # -rocksdb_block_cache_filter_miss # -rocksdb_block_cache_hit # -rocksdb_block_cache_index_add # -rocksdb_block_cache_index_bytes_evict # -rocksdb_block_cache_index_bytes_insert # -rocksdb_block_cache_index_hit # -rocksdb_block_cache_index_miss # -rocksdb_block_cache_miss # -rocksdb_block_cachecompressed_hit # -rocksdb_block_cachecompressed_miss # +Rocksdb_rows_deleted # +Rocksdb_rows_inserted # +Rocksdb_rows_read # +Rocksdb_rows_updated # +Rocksdb_rows_deleted_blind # +Rocksdb_rows_expired # +Rocksdb_rows_filtered # +Rocksdb_system_rows_deleted # +Rocksdb_system_rows_inserted # +Rocksdb_system_rows_read # +Rocksdb_system_rows_updated # +Rocksdb_memtable_total # +Rocksdb_memtable_unflushed # +Rocksdb_queries_point # +Rocksdb_queries_range # +Rocksdb_covered_secondary_key_lookups # +Rocksdb_block_cache_add # +Rocksdb_block_cache_add_failures # +Rocksdb_block_cache_bytes_read # +Rocksdb_block_cache_bytes_write # +Rocksdb_block_cache_data_add # +Rocksdb_block_cache_data_bytes_insert # +Rocksdb_block_cache_data_hit # +Rocksdb_block_cache_data_miss # +Rocksdb_block_cache_filter_add # +Rocksdb_block_cache_filter_bytes_evict # +Rocksdb_block_cache_filter_bytes_insert # +Rocksdb_block_cache_filter_hit # +Rocksdb_block_cache_filter_miss # +Rocksdb_block_cache_hit # +Rocksdb_block_cache_index_add # +Rocksdb_block_cache_index_bytes_evict # +Rocksdb_block_cache_index_bytes_insert # +Rocksdb_block_cache_index_hit # +Rocksdb_block_cache_index_miss # +Rocksdb_block_cache_miss # +Rocksdb_block_cachecompressed_hit # +Rocksdb_block_cachecompressed_miss # + rocksdb_bloom_filter_full_positive # + rocksdb_bloom_filter_full_true_positive # -rocksdb_bloom_filter_prefix_checked # -rocksdb_bloom_filter_prefix_useful # -rocksdb_bloom_filter_useful # -rocksdb_bytes_read # -rocksdb_bytes_written # -rocksdb_compact_read_bytes # -rocksdb_compact_write_bytes # -rocksdb_compaction_key_drop_new # -rocksdb_compaction_key_drop_obsolete # -rocksdb_compaction_key_drop_user # -rocksdb_flush_write_bytes # -rocksdb_get_hit_l0 # -rocksdb_get_hit_l1 # -rocksdb_get_hit_l2_and_up # -rocksdb_getupdatessince_calls # -rocksdb_git_date # -rocksdb_git_hash # -rocksdb_iter_bytes_read # +Rocksdb_bloom_filter_prefix_checked # +Rocksdb_bloom_filter_prefix_useful # +Rocksdb_bloom_filter_useful # +Rocksdb_bytes_read # +Rocksdb_bytes_written # +Rocksdb_compact_read_bytes # +Rocksdb_compact_write_bytes # +Rocksdb_compaction_key_drop_new # +Rocksdb_compaction_key_drop_obsolete # +Rocksdb_compaction_key_drop_user # +Rocksdb_flush_write_bytes # +Rocksdb_get_hit_l0 # +Rocksdb_get_hit_l1 # +Rocksdb_get_hit_l2_and_up # +Rocksdb_getupdatessince_calls # +Rocksdb_iter_bytes_read # +Rocksdb_memtable_hit # +Rocksdb_memtable_miss # +Rocksdb_no_file_closes # +Rocksdb_no_file_errors # + rocksdb_manual_compactions_processed # + rocksdb_manual_compactions_running # -rocksdb_memtable_hit # -rocksdb_memtable_miss # -rocksdb_no_file_closes # -rocksdb_no_file_errors # -rocksdb_no_file_opens # -rocksdb_num_iterators # -rocksdb_number_block_not_compressed # -rocksdb_number_db_next # -rocksdb_number_db_next_found # -rocksdb_number_db_prev # -rocksdb_number_db_prev_found # -rocksdb_number_db_seek # -rocksdb_number_db_seek_found # -rocksdb_number_deletes_filtered # -rocksdb_number_keys_read # -rocksdb_number_keys_updated # -rocksdb_number_keys_written # -rocksdb_number_merge_failures # -rocksdb_number_multiget_bytes_read # -rocksdb_number_multiget_get # -rocksdb_number_multiget_keys_read # -rocksdb_number_reseeks_iteration # -rocksdb_number_sst_entry_delete # -rocksdb_number_sst_entry_merge # -rocksdb_number_sst_entry_other # -rocksdb_number_sst_entry_put # -rocksdb_number_sst_entry_singledelete # -rocksdb_number_superversion_acquires # -rocksdb_number_superversion_cleanups # -rocksdb_number_superversion_releases # -rocksdb_row_lock_deadlocks # -rocksdb_row_lock_wait_timeouts # -rocksdb_snapshot_conflict_errors # -rocksdb_stall_l0_file_count_limit_slowdowns # -rocksdb_stall_locked_l0_file_count_limit_slowdowns # -rocksdb_stall_l0_file_count_limit_stops # -rocksdb_stall_locked_l0_file_count_limit_stops # -rocksdb_stall_pending_compaction_limit_stops # -rocksdb_stall_pending_compaction_limit_slowdowns # -rocksdb_stall_memtable_limit_stops # -rocksdb_stall_memtable_limit_slowdowns # -rocksdb_stall_total_stops # -rocksdb_stall_total_slowdowns # -rocksdb_stall_micros # -rocksdb_wal_bytes # -rocksdb_wal_group_syncs # -rocksdb_wal_synced # -rocksdb_write_other # -rocksdb_write_self # -rocksdb_write_timedout # -rocksdb_write_wal # +Rocksdb_no_file_opens # +Rocksdb_num_iterators # +Rocksdb_number_block_not_compressed # +Rocksdb_number_db_next # +Rocksdb_number_db_next_found # +Rocksdb_number_db_prev # +Rocksdb_number_db_prev_found # +Rocksdb_number_db_seek # +Rocksdb_number_db_seek_found # +Rocksdb_number_deletes_filtered # +Rocksdb_number_keys_read # +Rocksdb_number_keys_updated # +Rocksdb_number_keys_written # +Rocksdb_number_merge_failures # +Rocksdb_number_multiget_bytes_read # +Rocksdb_number_multiget_get # +Rocksdb_number_multiget_keys_read # +Rocksdb_number_reseeks_iteration # +Rocksdb_number_sst_entry_delete # +Rocksdb_number_sst_entry_merge # +Rocksdb_number_sst_entry_other # +Rocksdb_number_sst_entry_put # +Rocksdb_number_sst_entry_singledelete # +Rocksdb_number_superversion_acquires # +Rocksdb_number_superversion_cleanups # +Rocksdb_number_superversion_releases # +Rocksdb_row_lock_deadlocks # +Rocksdb_row_lock_wait_timeouts # +Rocksdb_snapshot_conflict_errors # +Rocksdb_stall_l0_file_count_limit_slowdowns # +Rocksdb_stall_locked_l0_file_count_limit_slowdowns # +Rocksdb_stall_l0_file_count_limit_stops # +Rocksdb_stall_locked_l0_file_count_limit_stops # +Rocksdb_stall_pending_compaction_limit_stops # +Rocksdb_stall_pending_compaction_limit_slowdowns # +Rocksdb_stall_memtable_limit_stops # +Rocksdb_stall_memtable_limit_slowdowns # +Rocksdb_stall_total_stops # +Rocksdb_stall_total_slowdowns # +Rocksdb_stall_micros # +Rocksdb_wal_bytes # +Rocksdb_wal_group_syncs # +Rocksdb_wal_synced # +Rocksdb_write_other # +Rocksdb_write_self # +Rocksdb_write_timedout # +Rocksdb_write_wal # select VARIABLE_NAME from INFORMATION_SCHEMA.global_status where VARIABLE_NAME LIKE 'rocksdb%'; VARIABLE_NAME ROCKSDB_ROWS_DELETED @@@ -1617,7 -1614,11 +1631,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES @@@ -1726,7 -1729,11 +1744,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result index 24466d9,0b1d346..b096e42 --- a/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result @@@ -1,11 -1,11 +1,11 @@@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] DROP TABLE IF EXISTS t1; +connection slave; include/stop_slave.inc + create table t1 (a int, b int, primary key (a), unique key (b)) engine=rocksdb; +connection master; - create table t1 (a int) engine=rocksdb; +connection slave; show variables like 'rpl_skip_tx_api'; Variable_name Value rpl_skip_tx_api ON diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test index d715eb7,522ff9f..4f34cbe --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test @@@ -1,6 -1,6 +1,7 @@@ --source include/have_rocksdb.inc --source include/have_debug.inc +--source include/have_partition.inc + --source include/not_valgrind.inc --disable_warnings drop table if exists t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test index 5eac859,be8b9fb..61a10a6 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test @@@ -66,9 -63,7 +66,9 @@@ ALTER TABLE t1 ADD INDEX kb(b), ALGORIT --disable_warnings # now do same index using copy algorithm # hitting max row locks (1M) +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; - --error ER_RDB_STATUS_GENERAL + --error ER_GET_ERRMSG ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; diff --cc storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test index 86a0a7f,238e62a..f4da0b7 --- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test @@@ -117,12 -115,12 +117,35 @@@ INSERT INTO t0 VALUES(0) ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +--echo #--------------------------------------------------------------- +--echo # MDEV-16703 Assertion failed in load_auto_incr_value_from_index +--echo #--------------------------------------------------------------- + +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++ ++--echo #---------------------------------- ++--echo # Issue #792 Crash in autoincrement ++--echo #---------------------------------- ++ ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++ ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; ++ + --echo #---------------------------------- + --echo # Issue #869 Crash in autoincrement + --echo #---------------------------------- + + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test index 9e7c816,c1cdf53..53bfba3 --- a/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test @@@ -8,7 -4,8 +8,10 @@@ let $restart_file= $MYSQLTEST_VARDIR/tm let $error_log= $MYSQLTEST_VARDIR/log/my_restart.err; select variable_name, variable_value from information_schema.global_variables where variable_name="rocksdb_ignore_unknown_options"; ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" +--exec find $MYSQLD_DATADIR/#rocksdb/OPTIONS* | sort -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++ --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --shutdown_server 10 diff --cc storage/rocksdb/mysql-test/rocksdb/t/collation.test index 7152925,38ce1b2..3b808bc --- a/storage/rocksdb/mysql-test/rocksdb/t/collation.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/collation.test @@@ -1,13 -1,7 +1,9 @@@ --source include/have_rocksdb.inc ---source include/have_fullregex.inc +# MariaDB doesn't have server variables to check for GCC version, so the +# following check is commented out: +# --source include/have_fullregex.inc - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - - --disable_warnings - DROP TABLE IF EXISTS t1; - --enable_warnings + call mtr.add_suppression("Invalid pattern"); # ci non-indexed column is allowed CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; diff --cc storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 67dae5d,2ae0b40..09998b9 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@@ -28,10 -17,10 +28,11 @@@ select VALUE into @keysIn from INFORMAT CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --cc storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test index b884738,cf8fb29..f4fdaa7 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test @@@ -1,5 -1,5 +1,6 @@@ --source include/have_rocksdb.inc +--source include/have_partition.inc + --source suite/rocksdb/include/have_write_committed.inc # # RocksDB Storage Engine tests diff --cc storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test index e9dcc60,d9ffb08..9f1e28d --- a/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test @@@ -43,41 -46,47 +45,44 @@@ shutdown_server 10 shutdown_server 10; # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message ---exec echo "Expect errors that we are missing two .frm files" ---let SEARCH_FILE=$LOG ---let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t1 is registered in RocksDB but does not have a .frm file +--echo "Expect errors that we are missing two .frm files" +--let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN=Schema mismatch --source include/search_pattern_in_file.inc + --let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t2 is registered in RocksDB but does not have a .frm file + --source include/search_pattern_in_file.inc # Now shut down again and rename one the .frm file back and make a copy of it --exec echo "wait" >$_expect_file_name shutdown_server 10; - + --remove_file $LOG - # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm ---exec cp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm +--copy_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message for the second case ---exec echo "Expect an error that we have an extra .frm file" +--echo "Expect an error that we have an extra .frm file" - --let SEARCH_PATTERN=Schema mismatch + --let SEARCH_FILE=$LOG + --let SEARCH_PATTERN=Schema mismatch - A .frm file exists for table test.t1_dummy, but that table is not registered in RocksDB --source include/search_pattern_in_file.inc # Shut down an clean up diff --cc storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc index 5e5961f,492b5cd..16ad535 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc @@@ -40,9 -40,9 +40,9 @@@ select * from mysql.slave_gtid_info --write_file $MYSQL_TMP_DIR/truncate_tail_wal.sh #!/bin/bash -F=`ls -t $slave_data_dir/.rocksdb/*.log | head -n 1` +F=`ls -t $slave_data_dir/\#rocksdb/*.log | head -n 1` SIZE=`stat -c %s $F` - NEW_SIZE=`expr $SIZE - 10` + NEW_SIZE=`expr $SIZE - 30` truncate -s $NEW_SIZE $F rc=$? if [[ $rc != 0 ]]; then diff --cc storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test index 39688e6,31b50aa..8d3bab4 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test @@@ -1,7 -1,21 +1,21 @@@ --source include/have_rocksdb.inc + CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO valid_values VALUES(65536); + INSERT INTO valid_values VALUES(1024); + INSERT INTO valid_values VALUES(1*1024*1024); + + CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO invalid_values VALUES('\'aaa\''); + INSERT INTO invalid_values VALUES('\'bbb\''); + INSERT INTO invalid_values VALUES('\'-1\''); + INSERT INTO invalid_values VALUES('\'101\''); + INSERT INTO invalid_values VALUES('\'484436\''); + --let $sys_var=ROCKSDB_BLOCK_CACHE_SIZE - --let $read_only=1 + --let $read_only=0 --let $session=0 ---source ../include/rocksdb_sys_var.inc +--source include/rocksdb_sys_var.inc + DROP TABLE valid_values; + DROP TABLE invalid_values; diff --cc storage/rocksdb/rdb_i_s.cc index de33d69,147ad7f..9192795 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@@ -1757,10 -1869,26 +1906,26 @@@ struct st_maria_plugin rdb_i_s_ddl = nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ + 0, /* flags */ + }; + + struct st_mysql_plugin rdb_i_s_sst_props = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_SST_PROPS", + "Facebook", + "RocksDB SST Properties", + PLUGIN_LICENSE_GPL, + rdb_i_s_sst_props_init, + rdb_i_s_deinit, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; -struct st_mysql_plugin rdb_i_s_index_file_map = { +struct st_maria_plugin rdb_i_s_index_file_map = { MYSQL_INFORMATION_SCHEMA_PLUGIN, &rdb_i_s_info, "ROCKSDB_INDEX_FILE_MAP", diff --cc storage/rocksdb/rdb_i_s.h index d6a48bf,dd9f4a9..f4c4b04 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@@ -21,16 -21,17 +21,17 @@@ namespace myrocks Declare INFORMATION_SCHEMA (I_S) plugins needed by MyRocks storage engine. */ -extern struct st_mysql_plugin rdb_i_s_cfstats; -extern struct st_mysql_plugin rdb_i_s_dbstats; -extern struct st_mysql_plugin rdb_i_s_perf_context; -extern struct st_mysql_plugin rdb_i_s_perf_context_global; -extern struct st_mysql_plugin rdb_i_s_cfoptions; -extern struct st_mysql_plugin rdb_i_s_compact_stats; -extern struct st_mysql_plugin rdb_i_s_global_info; -extern struct st_mysql_plugin rdb_i_s_ddl; -extern struct st_mysql_plugin rdb_i_s_sst_props; -extern struct st_mysql_plugin rdb_i_s_index_file_map; -extern struct st_mysql_plugin rdb_i_s_lock_info; -extern struct st_mysql_plugin rdb_i_s_trx_info; -extern struct st_mysql_plugin rdb_i_s_deadlock_info; +extern struct st_maria_plugin rdb_i_s_cfstats; +extern struct st_maria_plugin rdb_i_s_dbstats; +extern struct st_maria_plugin rdb_i_s_perf_context; +extern struct st_maria_plugin rdb_i_s_perf_context_global; +extern struct st_maria_plugin rdb_i_s_cfoptions; +extern struct st_maria_plugin rdb_i_s_compact_stats; +extern struct st_maria_plugin rdb_i_s_global_info; +extern struct st_maria_plugin rdb_i_s_ddl; ++extern struct st_maria_plugin rdb_i_s_sst_props; +extern struct st_maria_plugin rdb_i_s_index_file_map; +extern struct st_maria_plugin rdb_i_s_lock_info; +extern struct st_maria_plugin rdb_i_s_trx_info; +extern struct st_maria_plugin rdb_i_s_deadlock_info; } // namespace myrocks diff --cc storage/rocksdb/rocksdb index ba295cd,0000000..926f3a7 mode 160000,000000..160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@@ -1,1 -1,0 +1,1 @@@ - Subproject commit ba295cda29daee3ffe58549542804efdfd969784 ++Subproject commit 926f3a78a64b327475ee6c60b6c8ab4f34253204

1 0

[Commits] 7b1918e: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge
by psergey＠askmonty.org 28 Aug '18

28 Aug '18

revision-id: 7b1918ed47fd0c03b2ec7163e2272740423e9cfb parent(s): 55163ba1bdb1a05daadc66c41c959994231b361c faa4d8f8c6c12c09541c4733773ac700c5e643cf committer: Sergei Petrunia branch nick: 10.2-mariarocks timestamp: 2018-08-28 12:51:22 +0300 message: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge Move up-to this revision in the upstream: commit de1e8c7bfe7c875ea284b55040e8f3cd3a56fcc2 Author: Abhinav Sharma <abhinavsharma(a)fb.com> Date: Thu Aug 23 14:34:39 2018 -0700 Log updates to semi-sync whitelist in the error log Summary: Plugin variable changes are not logged in the error log even when log_global_var_changes is enabled. Logging updates to whitelist will help in debugging. Reviewed By: guokeno0 Differential Revision: D9483807 fbshipit-source-id: e111cda773d storage/rocksdb/CMakeLists.txt | 8 +- storage/rocksdb/ha_rocksdb.cc | 1219 +++++++++++++++----- storage/rocksdb/ha_rocksdb.h | 53 +- storage/rocksdb/ha_rocksdb_proto.h | 7 +- storage/rocksdb/mysql-test/rocksdb/combinations | 6 + .../rocksdb/include/have_write_committed.inc | 3 + .../mysql-test/rocksdb/r/add_index_inplace.result | 24 + .../r/add_index_inplace_sstfilewriter.result | 2 +- .../rocksdb/r/allow_no_primary_key.result | 6 +- .../mysql-test/rocksdb/r/autoinc_debug.result | 40 +- .../mysql-test/rocksdb/r/autoinc_vars.result | 18 + .../mysql-test/rocksdb/r/bloomfilter5.result | 4 +- .../rocksdb/r/bloomfilter_bulk_load.result | 15 + .../mysql-test/rocksdb/r/bulk_load_sk.result | 229 ++++ .../mysql-test/rocksdb/r/cardinality.result | 17 +- .../rocksdb/mysql-test/rocksdb/r/collation.result | 20 +- .../rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result | 21 + .../rocksdb/r/create_no_primary_key_table.result | 38 + .../mysql-test/rocksdb/r/ddl_high_priority.result | 71 +- .../mysql-test/rocksdb/r/deadlock_tracking.result | 57 + .../rocksdb/mysql-test/rocksdb/r/drop_table.result | 11 + .../mysql-test/rocksdb/r/explicit_snapshot.result | 265 +++++ .../mysql-test/rocksdb/r/i_s_deadlock.result | 55 +- .../mysql-test/rocksdb/r/index_file_map.result | 3 + .../rocksdb/r/index_merge_rocksdb2.result | 52 +- .../mysql-test/rocksdb/r/information_schema.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/lock.result | 15 + .../rocksdb/mysql-test/rocksdb/r/mysqldump.result | 5 +- .../r/percona_nonflushing_analyze_debug.result | 19 + .../rocksdb/r/prefix_extractor_override.result | 7 +- .../mysql-test/rocksdb/r/read_only_tx.result | 2 +- .../rocksdb/mysql-test/rocksdb/r/rocksdb.result | 24 +- .../mysql-test/rocksdb/r/rocksdb_checksums.result | 6 +- .../rocksdb/r/rocksdb_deadlock_detect_rc.result | 16 +- .../rocksdb/r/rocksdb_deadlock_detect_rr.result | 16 +- .../rocksdb/r/secondary_key_update_lock.result | 11 + .../mysql-test/rocksdb/r/show_engine.result | 53 + .../mysql-test/rocksdb/r/transaction.result | 17 + .../mysql-test/rocksdb/r/trx_info_rpl.result | 2 +- storage/rocksdb/mysql-test/rocksdb/r/update.result | 8 + .../mysql-test/rocksdb/r/use_direct_reads.result | 18 + .../mysql-test/rocksdb/t/add_index_inplace.test | 18 + .../rocksdb/t/add_index_inplace_crash.test | 1 + .../rocksdb/t/add_index_inplace_sstfilewriter.test | 2 +- .../mysql-test/rocksdb/t/autoinc_debug.test | 17 +- .../rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 23 + .../mysql-test/rocksdb/t/bloomfilter5-master.opt | 4 +- .../rocksdb/t/bloomfilter_bulk_load-master.opt | 2 + .../rocksdb/t/bloomfilter_bulk_load.test | 35 + .../rocksdb/mysql-test/rocksdb/t/bulk_load_sk.test | 119 ++ .../rocksdb/mysql-test/rocksdb/t/cardinality.test | 21 +- .../rocksdb/t/check_ignore_unknown_options.test | 3 + .../rocksdb/mysql-test/rocksdb/t/collation.test | 36 +- .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.cnf | 4 + .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.test | 87 ++ .../t/create_no_primary_key_table-master.opt | 1 + .../rocksdb/t/create_no_primary_key_table.test | 44 + .../mysql-test/rocksdb/t/deadlock_tracking.test | 76 +- .../rocksdb/mysql-test/rocksdb/t/drop_table.test | 27 + .../rocksdb/t/explicit_snapshot-master.opt | 1 + .../mysql-test/rocksdb/t/explicit_snapshot.test | 263 +++++ .../rocksdb/mysql-test/rocksdb/t/i_s_deadlock.test | 14 +- .../mysql-test/rocksdb/t/index_file_map.test | 3 + .../rocksdb/t/index_merge_rocksdb2-master.opt | 2 +- .../mysql-test/rocksdb/t/information_schema.test | 1 + .../rocksdb/t/insert_optimized_config-master.opt | 8 - .../rocksdb/t/insert_optimized_config.test | 10 + storage/rocksdb/mysql-test/rocksdb/t/lock.test | 22 + .../t/percona_nonflushing_analyze_debug.test | 11 + .../rocksdb/t/prefix_extractor_override.test | 14 +- storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test | 3 +- .../mysql-test/rocksdb/t/rocksdb_checksums.test | 10 +- .../rocksdb/t/rocksdb_deadlock_detect.inc | 35 +- .../rocksdb/t/secondary_key_update_lock.test | 26 + .../mysql-test/rocksdb/t/set_checkpoint.inc | 2 +- .../rocksdb/mysql-test/rocksdb/t/show_engine.test | 13 + .../rocksdb/mysql-test/rocksdb/t/transaction.test | 23 + .../rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test | 4 +- storage/rocksdb/mysql-test/rocksdb/t/update.test | 10 + .../mysql-test/rocksdb/t/use_direct_reads.test | 37 + .../mysql-test/rocksdb/t/validate_datadic.test | 16 +- .../rocksdb/mysql-test/rocksdb_rpl/combinations | 8 +- .../r/rpl_missing_columns_sk_update.result | 62 + .../rpl_mts_dependency_unique_key_conflicts.result | 44 + .../rocksdb_rpl/r/rpl_rocksdb_snapshot.result | 4 +- .../t/rpl_gtid_crash_safe_wal_corrupt.inc | 2 +- .../t/rpl_missing_columns_sk_update.cnf | 13 + .../t/rpl_missing_columns_sk_update.test | 69 ++ .../t/rpl_mts_dependency_unique_key_conflicts.test | 64 + .../t/rpl_rocksdb_2pc_crash_recover.test | 1 + .../rocksdb/mysql-test/rocksdb_stress/combinations | 5 + .../r/rocksdb_block_cache_size_basic.result | 84 +- .../r/rocksdb_bulk_load_allow_sk_basic.result | 100 ++ ...sdb_commit_time_batch_for_recovery_basic.result | 121 ++ .../r/rocksdb_create_checkpoint_basic.result | 2 +- ...ksdb_debug_manual_compaction_delay_basic.result | 46 + ...ksdb_error_on_suboptimal_collation_basic.result | 7 + .../rocksdb_manual_compaction_threads_basic.result | 93 ++ .../r/rocksdb_max_manifest_file_size_basic.result | 2 +- .../r/rocksdb_max_manual_compactions_basic.result | 57 + .../r/rocksdb_max_row_locks_basic.result | 21 + .../r/rocksdb_stats_recalc_rate_basic.result | 53 + .../r/rocksdb_write_batch_max_bytes_basic.result | 4 +- .../r/rocksdb_write_policy_basic.result | 15 + .../t/rocksdb_block_cache_size_basic.test | 16 +- .../t/rocksdb_bulk_load_allow_sk_basic.test | 18 + ...cksdb_commit_time_batch_for_recovery_basic.test | 20 + .../t/rocksdb_create_checkpoint_basic.test | 2 +- ...ocksdb_debug_manual_compaction_delay_basic.test | 16 + ...ocksdb_error_on_suboptimal_collation_basic.test | 6 + .../t/rocksdb_manual_compaction_threads_basic.test | 17 + .../t/rocksdb_max_manual_compactions_basic.test | 17 + .../t/rocksdb_max_row_locks_basic.test | 1 + .../t/rocksdb_stats_recalc_rate_basic.test | 17 + .../t/rocksdb_write_batch_max_bytes_basic.test | 4 +- .../t/rocksdb_write_policy_basic.test | 17 + storage/rocksdb/properties_collector.cc | 61 +- storage/rocksdb/properties_collector.h | 3 +- storage/rocksdb/rdb_cf_options.cc | 2 - storage/rocksdb/rdb_comparator.h | 46 +- storage/rocksdb/rdb_datadic.cc | 6 +- storage/rocksdb/rdb_datadic.h | 5 +- storage/rocksdb/rdb_i_s.cc | 167 ++- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_index_merge.cc | 5 + storage/rocksdb/rdb_perf_context.cc | 2 - storage/rocksdb/rdb_psi.cc | 17 +- storage/rocksdb/rdb_psi.h | 10 +- storage/rocksdb/rdb_sst_info.cc | 4 +- storage/rocksdb/rdb_threads.h | 31 +- storage/rocksdb/rdb_utils.cc | 23 - storage/rocksdb/rocksdb | 2 +- 132 files changed, 4286 insertions(+), 639 deletions(-) diff --cc storage/rocksdb/CMakeLists.txt index 2a4ea4d,54b0b1f..2689dc7 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@@ -130,55 -53,64 +130,58 @@@ if(HAVE_THREAD_LOCAL ADD_DEFINITIONS(-DROCKSDB_SUPPORT_THREAD_LOCAL) endif() -SET(ROCKSDB_SOURCES - ha_rocksdb.cc ha_rocksdb.h ha_rocksdb_proto.h +INCLUDE(build_rocksdb.cmake) + +ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib + ha_rocksdb_proto.h logger.h rdb_comparator.h - rdb_datadic.cc rdb_datadic.h - rdb_cf_options.cc rdb_cf_options.h - rdb_cf_manager.cc rdb_cf_manager.h - properties_collector.cc properties_collector.h - event_listener.cc event_listener.h - rdb_i_s.cc rdb_i_s.h - rdb_index_merge.cc rdb_index_merge.h - rdb_io_watchdog.cc rdb_io_watchdog.h - rdb_perf_context.cc rdb_perf_context.h - rdb_mutex_wrapper.cc rdb_mutex_wrapper.h - rdb_psi.h rdb_psi.cc - rdb_sst_info.cc rdb_sst_info.h - rdb_utils.cc rdb_utils.h rdb_buff.h - rdb_threads.cc rdb_threads.h - ${ROCKSDB_LIB_SOURCES} + rdb_cf_options.cc + rdb_cf_options.h + event_listener.cc + event_listener.h + rdb_perf_context.cc + rdb_perf_context.h + rdb_buff.h + rdb_mariadb_port.h ) -IF(WITH_FB_TSAN) - SET(PIC_EXT "_pic") -ELSE() - SET(PIC_EXT "") -ENDIF() +ADD_DEPENDENCIES(rocksdb_aux_lib GenError) -SET(rocksdb_static_libs ) -IF (WITH_SNAPPY) - FIND_LIBRARY(SNAPPY_LIBRARY - NAMES libsnappy${PIC_EXT}.a snappy - HINTS ${WITH_SNAPPY}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${SNAPPY_LIBRARY}) - ADD_DEFINITIONS(-DSNAPPY) -ENDIF() +# MARIAROCKS-TODO: how to properly depend on -lrt ? +TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY}) +if (UNIX AND NOT APPLE) + TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt) +endif() -IF (WITH_LZ4) +TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib) + FIND_LIBRARY(LZ4_LIBRARY + NAMES liblz4${PIC_EXT}.a lz4 + HINTS ${WITH_LZ4}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${LZ4_LIBRARY}) - ADD_DEFINITIONS(-DLZ4) -ENDIF() -IF (WITH_BZ2) - FIND_LIBRARY(BZ2_LIBRARY - NAMES libbz2${PIC_EXT}.a bz2 - HINTS ${WITH_BZ2}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${BZ2_LIBRARY}) - ADD_DEFINITIONS(-DBZIP2) +IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + + # MARIAROCKS_NOT_YET: Add -frtti flag when compiling RocksDB files. + # TODO: is this the right way to do this? + # - SQL layer and storage/rocksdb/*.cc are compiled with -fnortti + # - RocksDB files are compiled with "-fnortti ... -frtti" + # - This causes RocksDB headers to be compiled with different settings: + # = with RTTI when compiling RocksDB + # = without RTTI when compiling storage/rocksdb/*.cc + # + # (facebook/mysql-5.6 just compiles everything without -f*rtti, which means + # everything is compiled with -frtti) + # + # (also had to add -frtti above, because something that event_listener.cc + # includes requires it. So, now everything in MariaRocks is compiled with + # -frtti) + set_source_files_properties(event_listener.cc rdb_cf_options.cc rdb_sst_info.cc + PROPERTIES COMPILE_FLAGS -frtti) ENDIF() -IF (WITH_ZSTD) - SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZSTD_LIBRARY}) - ADD_DEFINITIONS(-DZSTD) +CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU) +IF(HAVE_SCHED_GETCPU) + ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1) ENDIF() IF (WITH_TBB) diff --cc storage/rocksdb/ha_rocksdb.cc index adef71e,c7d7cc0..36054ac --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@@ -506,77 -529,20 +545,85 @@@ static my_bool rocksdb_enable_bulk_load static my_bool rocksdb_print_snapshot_conflict_queries = 0; static my_bool rocksdb_large_prefix = 0; static my_bool rocksdb_allow_to_start_after_corruption = 0; +static char* rocksdb_git_hash; + +char *compression_types_val= + const_cast<char*>(get_rocksdb_supported_compression_types()); + static uint64_t rocksdb_write_policy = + rocksdb::TxnDBWritePolicy::WRITE_COMMITTED; + static my_bool rocksdb_error_on_suboptimal_collation = 1; + static uint32_t rocksdb_stats_recalc_rate = 0; + static uint32_t rocksdb_debug_manual_compaction_delay = 0; + static uint32_t rocksdb_max_manual_compactions = 0; std::atomic<uint64_t> rocksdb_row_lock_deadlocks(0); std::atomic<uint64_t> rocksdb_row_lock_wait_timeouts(0); std::atomic<uint64_t> rocksdb_snapshot_conflict_errors(0); std::atomic<uint64_t> rocksdb_wal_group_syncs(0); + std::atomic<uint64_t> rocksdb_manual_compactions_processed(0); + std::atomic<uint64_t> rocksdb_manual_compactions_running(0); + + +/* + Remove directory with files in it. + Used to remove checkpoint created by mariabackup. +*/ +#ifdef _WIN32 +#include <direct.h> /* unlink*/ +#ifndef F_OK +#define F_OK 0 +#endif +#endif + +static int rmdir_force(const char *dir) { + if (access(dir, F_OK)) + return true; + + char path[FN_REFLEN]; + char sep[] = {FN_LIBCHAR, 0}; + int err = 0; + + MY_DIR *dir_info = my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i = 0; i < dir_info->number_of_files; i++) { + FILEINFO *file = dir_info->dir_entry + i; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + err = my_delete(path, 0); + + if (err) { + break; + } + } + + my_dirend(dir_info); + + if (!err) + err = rmdir(dir); + + return (err == 0) ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; +} + + +static void rocksdb_remove_mariabackup_checkpoint( + my_core::THD *const, + struct st_mysql_sys_var *const , + void *const var_ptr, const void *const) { + std::string mariabackup_checkpoint_dir(rocksdb_datadir); + + mariabackup_checkpoint_dir.append("/mariabackup-checkpoint"); + + if (unlink(mariabackup_checkpoint_dir.c_str()) == 0) + return; + + rmdir_force(mariabackup_checkpoint_dir.c_str()); +} + + static std::unique_ptr<rocksdb::DBOptions> rdb_init_rocksdb_db_options(void) { auto o = std::unique_ptr<rocksdb::DBOptions>(new rocksdb::DBOptions()); @@@ -1077,9 -1076,11 +1165,11 @@@ static MYSQL_SYSVAR_INT(table_cache_num "DBOptions::table_cache_numshardbits for RocksDB", nullptr, nullptr, rocksdb_db_options->table_cache_numshardbits, - /* min */ 0, /* max */ INT_MAX, 0); + // LRUCache limits this to 19 bits, anything greater + // fails to create a cache and returns a nullptr + /* min */ 0, /* max */ 19, 0); -static MYSQL_SYSVAR_ULONG(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, +static MYSQL_SYSVAR_UINT64_T(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "DBOptions::WAL_ttl_seconds for RocksDB", nullptr, nullptr, rocksdb_db_options->WAL_ttl_seconds, @@@ -1753,7 -1772,11 +1873,12 @@@ static struct st_mysql_sys_var *rocksdb MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(allow_to_start_after_corruption), + MYSQL_SYSVAR(git_hash), + MYSQL_SYSVAR(error_on_suboptimal_collation), + MYSQL_SYSVAR(stats_recalc_rate), + MYSQL_SYSVAR(debug_manual_compaction_delay), + MYSQL_SYSVAR(max_manual_compactions), + MYSQL_SYSVAR(manual_compaction_threads), nullptr}; static rocksdb::WriteOptions @@@ -1897,9 -1941,7 +2064,9 @@@ private ulonglong m_writes_at_last_savepoint; protected: - THD *m_thd = nullptr; + - rocksdb::ReadOptions m_read_opts; ++protected: + THD *m_thd = nullptr; static std::multiset<Rdb_transaction *> s_tx_list; static mysql_mutex_t s_tx_list_mutex; @@@ -1956,14 -1990,20 +2123,23 @@@ protected return s; } + protected: + /* + The following two are helper functions to be overloaded by child classes. + They should provide RocksDB's savepoint semantics. + */ + virtual void do_set_savepoint() = 0; + virtual void do_rollback_to_savepoint() = 0; + public: + rocksdb::ReadOptions m_read_opts; const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; +#ifdef MARIAROCKS_NOT_YET + // TODO: MariaDB probably doesn't need these at all: const char *m_mysql_gtid; const char *m_mysql_max_gtid; +#endif String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; @@@ -2946,6 -2991,6 +3141,7 @@@ error m_batch->RollbackToSavePoint(); } ++ public: bool is_writebatch_trx() const override { return true; } @@@ -3563,7 -3494,8 +3766,8 @@@ static int rocksdb_commit(handlerton* h /* We get here when committing a statement within a transaction. */ - tx->set_tx_failed(false); + tx->make_stmt_savepoint_permanent(); + tx->make_stmt_savepoint_permanent(); } if (my_core::thd_tx_isolation(thd) <= ISO_READ_COMMITTED) { @@@ -4119,9 -4052,24 +4326,25 @@@ static bool rocksdb_show_status(handler str, stat_print); } } - #endif + /* Explicit snapshot information */ + str.clear(); + { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + for (const auto &elem : explicit_snapshots) { + const auto &ss = elem.second.lock(); + DBUG_ASSERT(ss != nullptr); + const auto &info = ss->ss_info; + str += "\nSnapshot ID: " + std::to_string(info.snapshot_id) + + "\nBinlog File: " + info.binlog_file + + "\nBinlog Pos: " + std::to_string(info.binlog_pos) + + "\nGtid Executed: " + info.gtid_executed + "\n"; + } + } + if (!str.empty()) { + res |= print_stats(thd, "EXPLICIT_SNAPSHOTS", "rocksdb", str, stat_print); + } +#ifdef MARIAROCKS_NOT_YET } else if (stat_type == HA_ENGINE_TRX) { /* Handle the SHOW ENGINE ROCKSDB TRANSACTION STATUS command */ res |= rocksdb_show_snapshot_status(hton, thd, stat_print); @@@ -4141,8 -4089,48 +4364,50 @@@ static inline void rocksdb_register_tx( } } +static const char *ha_rocksdb_exts[] = {NullS}; + + static bool rocksdb_explicit_snapshot( + handlerton *const /* hton */, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle */ + snapshot_info_st *ss_info) /*!< out: Snapshot information */ + { + switch (ss_info->op) { + case snapshot_operation::SNAPSHOT_CREATE: { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + auto s = Rdb_explicit_snapshot::create(ss_info, rdb, rdb->GetSnapshot()); + if (mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + thd->set_explicit_snapshot(s); + return s == nullptr; + } + case snapshot_operation::SNAPSHOT_ATTACH: { + auto s = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!s) { + return true; + } + *ss_info = s->ss_info; + thd->set_explicit_snapshot(s); + return false; + } + case snapshot_operation::SNAPSHOT_RELEASE: { + if (!thd->get_explicit_snapshot()) { + return true; + } + *ss_info = thd->get_explicit_snapshot()->ss_info; + thd->set_explicit_snapshot(nullptr); + return false; + } + default: + DBUG_ASSERT(false); + return true; + } + return true; + } + /* Supporting START TRANSACTION WITH CONSISTENT [ROCKSDB] SNAPSHOT @@@ -4176,16 -4166,14 +4443,27 @@@ static int rocksdb_start_tx_and_assign_ my_error(ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT, MYF(0)); return HA_EXIT_FAILURE; } + ++#ifdef MARIADB_NOT_YET + if (ss_info) { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } else { + return HA_EXIT_FAILURE; + } - } ++#endif ++ + /* + MariaDB: there is no need to call mysql_bin_log_lock_commits and then + unlock back. + SQL layer calls start_consistent_snapshot() for all engines, including the + binlog under LOCK_commit_ordered mutex. ++ + The mutex prevents binlog commits from happening (right?) while the storage + engine(s) allocate read snapshots. That way, each storage engine is - synchronized with current binlog position. ++ synchronized with current binlog position. + */ + mysql_mutex_assert_owner(&LOCK_commit_ordered); Rdb_transaction *const tx = get_or_create_tx(thd); Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); @@@ -4195,6 -4183,10 +4473,11 @@@ rocksdb_register_tx(hton, thd, tx); tx->acquire_snapshot(true); ++#ifdef MARIADB_NOT_YET + if (ss_info) { + mysql_bin_log_unlock_commits(ss_info); + } - ++#endif return HA_EXIT_SUCCESS; } @@@ -4445,6 -4502,9 +4811,8 @@@ static int rocksdb_init_func(void *cons mysql_mutex_init(rdb_sysvars_psi_mutex_key, &rdb_sysvars_mutex, MY_MUTEX_INIT_FAST); + mysql_mutex_init(rdb_block_cache_resize_mutex_key, + &rdb_block_cache_resize_mutex, MY_MUTEX_INIT_FAST); - rdb_open_tables.init_hash(); Rdb_transaction::init_mutex(); rocksdb_hton->state = SHOW_OPTION_YES; @@@ -4457,16 -4514,14 +4825,18 @@@ rocksdb_hton->commit_by_xid = rocksdb_commit_by_xid; rocksdb_hton->rollback_by_xid = rocksdb_rollback_by_xid; rocksdb_hton->recover = rocksdb_recover; + + rocksdb_hton->commit_ordered= rocksdb_commit_ordered; rocksdb_hton->commit = rocksdb_commit; + + rocksdb_hton->commit_checkpoint_request= rocksdb_checkpoint_request; + rocksdb_hton->rollback = rocksdb_rollback; - rocksdb_hton->db_type = DB_TYPE_ROCKSDB; rocksdb_hton->show_status = rocksdb_show_status; + rocksdb_hton->explicit_snapshot = rocksdb_explicit_snapshot; rocksdb_hton->start_consistent_snapshot = rocksdb_start_tx_and_assign_read_view; + rocksdb_hton->start_shared_snapshot = rocksdb_start_tx_with_shared_read_view; rocksdb_hton->savepoint_set = rocksdb_savepoint; rocksdb_hton->savepoint_rollback = rocksdb_rollback_to_savepoint; rocksdb_hton->savepoint_rollback_can_release_mdl = @@@ -4710,9 -4791,10 +5111,9 @@@ } cf_manager.init(std::move(cf_options_map), &cf_handles); - if (dict_manager.init(rdb->GetBaseDB(), &cf_manager)) { + if (dict_manager.init(rdb, &cf_manager)) { // NO_LINT_DEBUG sql_print_error("RocksDB: Failed to initialize data dictionary."); - rdb_open_tables.free_hash(); DBUG_RETURN(HA_EXIT_FAILURE); } @@@ -4877,66 -4968,28 +5293,77 @@@ static int rocksdb_done_func(void *cons sql_print_error("RocksDB: Couldn't stop the index thread: (errno=%d)", err); } + // signal the manual compaction thread to stop + rdb_mc_thread.signal(true); + // Wait for the manual compaction thread to finish. + err = rdb_mc_thread.join(); + if (err != 0) { + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Couldn't stop the manual compaction thread: (errno=%d)", err); + } + - if (rdb_open_tables.m_hash.records) { + if (rdb_open_tables.m_hash.size()) { // Looks like we are getting unloaded and yet we have some open tables // left behind. error = 1; } - rdb_open_tables.free_hash(); + /* + MariaDB: When the plugin is unloaded with UNINSTALL SONAME command, some + connections may still have Rdb_transaction objects. + + These objects are not genuine transactions (as SQL layer makes sure that + a plugin that is being unloaded has no open tables), they are empty + Rdb_transaction objects that were left there to save on object + creation/deletion. + + Go through the list and delete them. + */ + { + class Rdb_trx_deleter: public Rdb_tx_list_walker { + public: + std::set<Rdb_transaction*> rdb_trxs; + + void process_tran(const Rdb_transaction *const tx) override { + /* + Check if the transaction is really empty. We only check + non-WriteBatch-based transactions, because there is no easy way to + check WriteBatch-based transactions. + */ + if (!tx->is_writebatch_trx()) { + const auto tx_impl = static_cast<const Rdb_transaction_impl *>(tx); + DBUG_ASSERT(tx_impl); + if (tx_impl->get_rdb_trx()) + DBUG_ASSERT(0); + } + rdb_trxs.insert((Rdb_transaction*)tx); + }; + } deleter; + + Rdb_transaction::walk_tx_list(&deleter); + + for (std::set<Rdb_transaction*>::iterator it= deleter.rdb_trxs.begin(); + it != deleter.rdb_trxs.end(); + ++it) + { + // When a transaction is deleted, it removes itself from s_tx_list. + delete *it; + } + } + + /* + destructors for static objects can be called at _exit(), + but we want to free the memory at dlclose() + */ + rdb_open_tables.m_hash.~Rdb_table_set(); mysql_mutex_destroy(&rdb_open_tables.m_mutex); mysql_mutex_destroy(&rdb_sysvars_mutex); + mysql_mutex_destroy(&rdb_block_cache_resize_mutex); + delete rdb_collation_exceptions; + mysql_mutex_destroy(&rdb_collation_data_mutex); mysql_mutex_destroy(&rdb_mem_cmp_space_mutex); @@@ -6307,14 -6348,13 +6729,14 @@@ void ha_rocksdb::free_key_buffers() my_free(m_dup_sk_packed_tuple_old); m_dup_sk_packed_tuple_old = nullptr; - my_free(m_eq_cond_upper_bound); - m_eq_cond_upper_bound = nullptr; + my_free(m_scan_it_lower_bound); + m_scan_it_lower_bound = nullptr; - my_free(m_eq_cond_lower_bound); - m_eq_cond_lower_bound = nullptr; + my_free(m_scan_it_upper_bound); + m_scan_it_upper_bound = nullptr; } +#ifdef MARIAROCKS_NOT_YET void ha_rocksdb::set_use_read_free_rpl(const char *const whitelist) { DBUG_ASSERT(whitelist != nullptr); @@@ -6796,12 -6766,12 +7216,10 @@@ int ha_rocksdb::create_cfs std::array<struct key_def_cf_info, MAX_INDEXES + 1> *const cfs) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); char tablename_sys[NAME_LEN + 1]; - - my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), - tablename_sys, sizeof(tablename_sys)); + bool tsys_set= false; /* The first loop checks the index parameters and creates @@@ -6810,40 -6780,37 +7228,49 @@@ for (uint i = 0; i < tbl_def_arg->m_key_count; i++) { rocksdb::ColumnFamilyHandle *cf_handle; - if (rocksdb_strict_collation_check && - !is_hidden_pk(i, table_arg, tbl_def_arg) && + if (!is_hidden_pk(i, table_arg, tbl_def_arg) && tbl_def_arg->base_tablename().find(tmp_file_prefix) != 0) { - for (uint part = 0; part < table_arg->key_info[i].actual_key_parts; - part++) { - if (!rdb_is_index_collation_supported( + if (!tsys_set) + { + tsys_set= true; + my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), + tablename_sys, sizeof(tablename_sys)); + } + + for (uint part = 0; part < table_arg->key_info[i].ext_key_parts; + part++) + { + /* MariaDB: disallow NOPAD collations */ + if (rdb_field_uses_nopad_collation( + table_arg->key_info[i].key_part[part].field)) + { + my_error(ER_MYROCKS_CANT_NOPAD_COLLATION, MYF(0)); + DBUG_RETURN(HA_EXIT_FAILURE); + } + + if (rocksdb_strict_collation_check && + !rdb_is_index_collation_supported( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - std::string collation_err; - for (const auto &coll : RDB_INDEX_COLLATIONS) { - if (collation_err != "") { - collation_err += ", "; - } - collation_err += coll->name; - } - if (rocksdb_error_on_suboptimal_collation) { - my_error(ER_UNSUPPORTED_COLLATION, MYF(0), - tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name, - collation_err.c_str()); - DBUG_RETURN(HA_EXIT_FAILURE); + char buf[1024]; + my_snprintf(buf, sizeof(buf), + "Indexed column %s.%s uses a collation that does not " + "allow index-only access in secondary key and has " + "reduced disk space efficiency in primary key.", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name); + + my_error(ER_INTERNAL_ERROR, MYF(ME_JUST_WARNING), buf); + } else { + push_warning_printf( + ha_thd(), Sql_condition::WARN_LEVEL_WARN, ER_WRONG_ARGUMENTS, + "Unsupported collation on string indexed column %s.%s Use " + "binary collation (%s).", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name, + collation_err.c_str()); + } } } } @@@ -7212,10 -7163,7 +7623,8 @@@ int ha_rocksdb::create_key_def(const TA int rdb_normalize_tablename(const std::string &tablename, std::string *const strbuf) { - DBUG_ASSERT(strbuf != nullptr); - - if (tablename.size() < 2 || tablename[0] != '.' || tablename[1] != '/') { + if (tablename.size() < 2 || tablename[0] != '.' || + (tablename[1] != FN_LIBCHAR && tablename[1] != FN_LIBCHAR2)) { DBUG_ASSERT(0); // We were not passed table name? return HA_ERR_ROCKSDB_INVALID_TABLE; } @@@ -7875,11 -7809,10 +8278,10 @@@ ulong ha_rocksdb::index_flags(uint inx pair for. */ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { - DBUG_ASSERT(buf != nullptr); DBUG_ASSERT(table != nullptr); - +#ifdef MARIAROCKS_NOT_YET stats.rows_requested++; - +#endif /* Use STATUS_NOT_FOUND when record not found or some error occurred */ table->status = STATUS_NOT_FOUND; @@@ -9798,34 -9666,23 +10187,28 @@@ int ha_rocksdb::update_write_row(const If the index was reverse order, upper bound would be 0x0000b3eb003f65c5e78857, and lower bound would be 0x0000b3eb003f65c5e78859. These cover given eq condition range. + + @param lower_bound_buf IN Buffer for lower bound + @param upper_bound_buf IN Buffer for upper bound + + @param outer_u */ - void ha_rocksdb::setup_iterator_bounds(const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { - uint eq_cond_len = eq_cond.size(); - memcpy(upper_bound_buf, eq_cond.data(), eq_cond_len); - kd.successor(upper_bound_buf, eq_cond_len); - memcpy(lower_bound_buf, eq_cond.data(), eq_cond_len); - kd.predecessor(lower_bound_buf, eq_cond_len); + void ha_rocksdb::setup_iterator_bounds( + const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, size_t bound_len, + uchar *const lower_bound, uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, rocksdb::Slice *upper_bound_slice) { + uint min_len = std::min(eq_cond.size(), bound_len); + memcpy(upper_bound, eq_cond.data(), min_len); + kd.successor(upper_bound, min_len); + memcpy(lower_bound, eq_cond.data(), min_len); + kd.predecessor(lower_bound, min_len); if (kd.m_is_reverse_cf) { - *out_upper_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); } else { - *out_upper_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); } } @@@ -9845,11 -9701,10 +10227,11 @@@ void ha_rocksdb::setup_scan_iterator(co bool skip_bloom = true; const rocksdb::Slice eq_cond(slice->data(), eq_cond_len); - if (check_bloom_and_set_bounds(ha_thd(), kd, eq_cond, use_all_keys, - m_eq_cond_lower_bound, - m_eq_cond_upper_bound, - &m_eq_cond_lower_bound_slice, + if (check_bloom_and_set_bounds( + ha_thd(), kd, eq_cond, use_all_keys, eq_cond_len, + m_scan_it_lower_bound, m_scan_it_upper_bound, + &m_scan_it_lower_bound_slice, &m_scan_it_upper_bound_slice)) { + &m_eq_cond_upper_bound_slice)) { skip_bloom = false; } @@@ -9955,9 -9810,10 +10337,9 @@@ int ha_rocksdb::rnd_next(uchar *const b DBUG_ENTER_FUNC(); int rc; - ha_statistic_increment(&SSV::ha_read_rnd_next_count); for (;;) { rc = rnd_next_with_direction(buf, true); - if (rc != HA_ERR_LOCK_DEADLOCK || !m_rnd_scan_is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !m_rnd_scan_is_new_snapshot) break; /* exit the loop */ // release the snapshot and iterator and then regenerate them @@@ -10362,12 -10214,23 +10744,23 @@@ int ha_rocksdb::info(uint flag) continue; } KEY *const k = &table->key_info[i]; - for (uint j = 0; j < k->actual_key_parts; j++) { + for (uint j = 0; j < k->ext_key_parts; j++) { const Rdb_index_stats &k_stats = m_key_descr_arr[i]->m_stats; - uint x = k_stats.m_distinct_keys_per_prefix.size() > j && - k_stats.m_distinct_keys_per_prefix[j] > 0 - ? k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j] - : 0; + uint x; + + if (k_stats.m_distinct_keys_per_prefix.size() > j && + k_stats.m_distinct_keys_per_prefix[j] > 0) { + x = k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j]; + /* + If the number of rows is less than the number of prefixes (due to + sampling), the average number of rows with the same prefix is 1. + */ + if (x == 0) { + x = 1; + } + } else { + x = 0; + } if (x > stats.records) x = stats.records; if ((x == 0 && rocksdb_debug_optimizer_no_zero_cardinality) || @@@ -12807,24 -12661,198 +13225,195 @@@ void Rdb_background_thread::run() rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); } } - + // Recalculate statistics for indexes. + if (rocksdb_stats_recalc_rate) { + std::unordered_map<GL_INDEX_ID, std::shared_ptr<const Rdb_key_def>> + to_recalc; + + if (rdb_indexes_to_recalc.empty()) { + struct Rdb_index_collector : public Rdb_tables_scanner { + int add_table(Rdb_tbl_def *tdef) override { + for (uint i = 0; i < tdef->m_key_count; i++) { + rdb_indexes_to_recalc.push_back( + tdef->m_key_descr_arr[i]->get_gl_index_id()); + } + return HA_EXIT_SUCCESS; + } + } collector; + ddl_manager.scan_for_tables(&collector); + } + + while (to_recalc.size() < rocksdb_stats_recalc_rate && + !rdb_indexes_to_recalc.empty()) { + const auto index_id = rdb_indexes_to_recalc.back(); + rdb_indexes_to_recalc.pop_back(); + + std::shared_ptr<const Rdb_key_def> keydef = + ddl_manager.safe_find(index_id); + + if (keydef) { + to_recalc.insert(std::make_pair(keydef->get_gl_index_id(), keydef)); + } + } + + if (!to_recalc.empty()) { + calculate_stats(to_recalc, false); + } + } + - // Set the next timestamp for mysql_cond_timedwait() (which ends up calling - // pthread_cond_timedwait()) to wait on. - ts_next_sync.tv_sec = ts.tv_sec + WAKE_UP_INTERVAL; } // save remaining stats which might've left unsaved ddl_manager.persist_stats(); } - bool ha_rocksdb::check_bloom_and_set_bounds(THD *thd, const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - const bool use_all_keys, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { + /* + A background thread to handle manual compactions, + except for dropping indexes/tables. Every second, it checks + pending manual compactions, and it calls CompactRange if there is. + */ + void Rdb_manual_compaction_thread::run() { + mysql_mutex_init(0, &m_mc_mutex, MY_MUTEX_INIT_FAST); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + for (;;) { + if (m_stop) { + break; + } + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 1; + + const auto ret MY_ATTRIBUTE((__unused__)) = + mysql_cond_timedwait(&m_signal_cond, &m_signal_mutex, &ts); + if (m_stop) { + break; + } + // make sure, no program error is returned + DBUG_ASSERT(ret == 0 || ret == ETIMEDOUT); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + // Grab the first item and proceed, if not empty. + if (m_requests.empty()) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + continue; + } + Manual_compaction_request &mcr = m_requests.begin()->second; + DBUG_ASSERT(mcr.cf != nullptr); + DBUG_ASSERT(mcr.state == Manual_compaction_request::INITED); + mcr.state = Manual_compaction_request::RUNNING; + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + + DBUG_ASSERT(mcr.state == Manual_compaction_request::RUNNING); + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s started.", mcr.mc_id, + mcr.cf->GetName().c_str()); + rocksdb_manual_compactions_running++; + if (rocksdb_debug_manual_compaction_delay > 0) { + my_sleep(rocksdb_debug_manual_compaction_delay * 1000000); + } + // CompactRange may take a very long time. On clean shutdown, + // it is cancelled by CancelAllBackgroundWork, then status is + // set to shutdownInProgress. + const rocksdb::Status s = rdb->CompactRange( + getCompactRangeOptions(mcr.concurrency), mcr.cf, mcr.start, mcr.limit); + rocksdb_manual_compactions_running--; + if (s.ok()) { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s ended.", mcr.mc_id, + mcr.cf->GetName().c_str()); + } else { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s aborted. %s", + mcr.mc_id, mcr.cf->GetName().c_str(), s.getState()); + if (!s.IsShutdownInProgress()) { + rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + } else { + DBUG_ASSERT(m_requests.size() == 1); + } + } + rocksdb_manual_compactions_processed++; + clear_manual_compaction_request(mcr.mc_id, false); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + } + clear_all_manual_compaction_requests(); + DBUG_ASSERT(m_requests.empty()); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + mysql_mutex_destroy(&m_mc_mutex); + } + + void Rdb_manual_compaction_thread::clear_all_manual_compaction_requests() { + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + m_requests.clear(); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + } + + void Rdb_manual_compaction_thread::clear_manual_compaction_request( + int mc_id, bool init_only) { + bool erase = true; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + if (init_only) { + Manual_compaction_request mcr = it->second; + if (mcr.state != Manual_compaction_request::INITED) { + erase = false; + } + } + if (erase) { + m_requests.erase(it); + } + } else { + // Current code path guarantees that erasing by the same mc_id happens + // at most once. INITED state may be erased by a thread that requested + // the compaction. RUNNING state is erased by mc thread only. + DBUG_ASSERT(0); + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + } + + int Rdb_manual_compaction_thread::request_manual_compaction( + rocksdb::ColumnFamilyHandle *cf, rocksdb::Slice *start, + rocksdb::Slice *limit, int concurrency) { + int mc_id = -1; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.size() >= rocksdb_max_manual_compactions) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + Manual_compaction_request mcr; + mc_id = mcr.mc_id = ++m_latest_mc_id; + mcr.state = Manual_compaction_request::INITED; + mcr.cf = cf; + mcr.start = start; + mcr.limit = limit; + mcr.concurrency = concurrency; + m_requests.insert(std::make_pair(mcr.mc_id, mcr)); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + + bool Rdb_manual_compaction_thread::is_manual_compaction_finished(int mc_id) { + bool finished = false; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.count(mc_id) == 0) { + finished = true; + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return finished; + } + + bool ha_rocksdb::check_bloom_and_set_bounds( + THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, + const bool use_all_keys, size_t bound_len, uchar *const lower_bound, + uchar *const upper_bound, rocksdb::Slice *lower_bound_slice, + rocksdb::Slice *upper_bound_slice) { bool can_use_bloom = can_use_bloom_filter(thd, kd, eq_cond, use_all_keys); if (!can_use_bloom) { - setup_iterator_bounds(kd, eq_cond, - lower_bound_buf, upper_bound_buf, + setup_iterator_bounds(kd, eq_cond, bound_len, lower_bound, upper_bound, + lower_bound_slice, upper_bound_slice); + out_lower_bound, out_upper_bound); } return can_use_bloom; } @@@ -12973,11 -13000,6 +13561,8 @@@ const char *get_rdb_io_error_string(con // In case of core dump generation we want this function NOT to be optimized // so that we can capture as much data as possible to debug the root cause // more efficiently. +#ifdef __GNUC__ - #pragma GCC push_options - #pragma GCC optimize("O0") +#endif - void rdb_handle_io_error(const rocksdb::Status status, const RDB_IO_ERROR_TYPE err_type) { if (status.IsIOError()) { @@@ -13026,10 -13051,7 +13614,8 @@@ } } } - +#ifdef __GNUC__ - #pragma GCC pop_options +#endif - Rdb_dict_manager *rdb_get_dict_manager(void) { return &dict_manager; } Rdb_ddl_manager *rdb_get_ddl_manager(void) { return &ddl_manager; } @@@ -13573,7 -13605,6 +14202,7 @@@ maria_declare_plugin(rocksdb_se) myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, myrocks::rdb_i_s_cfoptions, myrocks::rdb_i_s_compact_stats, myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, - myrocks::rdb_i_s_index_file_map, myrocks::rdb_i_s_lock_info, - myrocks::rdb_i_s_trx_info, + myrocks::rdb_i_s_sst_props, myrocks::rdb_i_s_index_file_map, + myrocks::rdb_i_s_lock_info, myrocks::rdb_i_s_trx_info, - myrocks::rdb_i_s_deadlock_info mysql_declare_plugin_end; + myrocks::rdb_i_s_deadlock_info +maria_declare_plugin_end; diff --cc storage/rocksdb/ha_rocksdb.h index 739a2c8,ca14165..c444710 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@@ -1402,8 -1369,7 +1407,8 @@@ public virtual void rpl_after_delete_rows() override; virtual void rpl_before_update_rows() override; virtual void rpl_after_update_rows() override; - virtual bool use_read_free_rpl(); + virtual bool use_read_free_rpl() override; +#endif // MARIAROCKS_NOT_YET private: /* Flags tracking if we are inside different replication operation */ diff --cc storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result index 08f2329,9731007..0617232 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result @@@ -6,18 -6,16 +6,18 @@@ LOAD DATA INFILE <input_file> INTO TABL set rocksdb_bulk_load=0; select count(pk) from t1; count(pk) -3000000 +300000 select count(a) from t1; count(a) -3000000 +300000 select count(b) from t1; count(b) -3000000 +300000 ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; - ERROR HY000: Status error 10 received from RocksDB: Operation aborted: Failed to acquire lock due to max_num_locks limit + ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to max_num_locks limit' from ROCKSDB set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=0; diff --cc storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result index a8d5c07,6f811e1..de8a1fb --- a/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result @@@ -276,16 -276,12 +276,16 @@@ 10 20 30 - connect con1,localhost,root,,; - connection con1; alter table t1 force; - connection default; + select * from t1; + a + insert into t1 values (100); select * from t1; a +connection con1; +insert into t1 values (100); +select * from t1; +a 1 2 100 diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result index fe08cd7,8b51c12..604e557 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result @@@ -56,52 -56,44 +56,44 @@@ insert into t values () begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_before"; +set debug_dbug="+d,crash_commit_before"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After engine prepare begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_prepare"; +set debug_dbug="+d,crash_commit_after_prepare"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After binlog begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_log"; +set debug_dbug="+d,crash_commit_after_log"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 6 - select max(i) from t; - max(i) - 5 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After everything begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after"; +set debug_dbug="+d,crash_commit_after"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 8 - select max(i) from t; - max(i) - 7 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 0c49622,17e6bed..f59b841 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@@ -150,11 -151,11 +150,29 @@@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +#--------------------------------------------------------------- +# MDEV-16703 Assertion failed in load_auto_incr_value_from_index +#--------------------------------------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++#---------------------------------- ++# Issue #792 Crash in autoincrement ++#---------------------------------- ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; + #---------------------------------- + # Issue #869 Crash in autoincrement + #---------------------------------- + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/collation.result index e372cbe,b4354ad..10e0d9b --- a/storage/rocksdb/mysql-test/rocksdb/r/collation.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/collation.result @@@ -1,11 -1,12 +1,12 @@@ - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - DROP TABLE IF EXISTS t1; + call mtr.add_suppression("Invalid pattern"); CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). DROP TABLE t1; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (latin1_bin, binary, utf8_bin). CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (latin1_bin, binary, utf8_bin). SET GLOBAL rocksdb_strict_collation_check=0; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; DROP TABLE t1; @@@ -124,7 -125,19 +126,19 @@@ ERROR HY000: Unsupported collation on s SET GLOBAL rocksdb_strict_collation_exceptions="abc"; CREATE TABLE abc (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; CREATE TABLE abcd (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (latin1_bin, binary, utf8_bin). DROP TABLE abc; SET GLOBAL rocksdb_strict_collation_exceptions=null; - SET GLOBAL rocksdb_strict_collation_exceptions=@start_global_value; + SET GLOBAL rocksdb_strict_collation_check=1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset latin1 collate latin1_bin; + ALTER TABLE t1 collate=latin1_general_ci; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result index 36db920,2a00fcb..0619e2d --- a/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result @@@ -13,8 -9,9 +13,9 @@@ connection default show create table information_schema.rocksdb_deadlock; Table Create Table ROCKSDB_DEADLOCK CREATE TEMPORARY TABLE `ROCKSDB_DEADLOCK` ( - `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT '0', + `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT 0, + `TIMESTAMP` bigint(8) NOT NULL DEFAULT '0', - `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT '0', + `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT 0, `CF_NAME` varchar(193) NOT NULL DEFAULT '', `WAITING_KEY` varchar(513) NOT NULL DEFAULT '', `LOCK_TYPE` varchar(193) NOT NULL DEFAULT '', @@@ -25,9 -22,8 +26,9 @@@ create table t (i int primary key) engine=rocksdb; insert into t values (1), (2), (3); select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #1 +connection con1; begin; select * from t where i=1 for update; i @@@ -47,13 -39,11 +48,13 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 Deadlock #2 +connection con1; begin; select * from t where i=1 for update; i @@@ -73,16 -59,14 +74,16 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 10; Deadlock #3 +connection con1; begin; select * from t where i=1 for update; i @@@ -102,24 -82,21 +103,24 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 1; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 +connection con3; set rocksdb_deadlock_detect_depth = 2; Deadlock #4 +connection con1; begin; select * from t where i=1 for update; i @@@ -150,12 -120,10 +151,12 @@@ connection con1 i 2 rollback; +connection default; set global rocksdb_max_latest_deadlocks = 5; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #5 +connection con1; begin; select * from t where i=1 for update; i @@@ -191,16 -154,11 +192,16 @@@ connection con1 i 2 rollback; +connection con3; rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 +disconnect con1; +disconnect con2; +disconnect con3; set global rocksdb_lock_wait_timeout = @prior_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_deadlock_detect; drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result index 4370715,80d95e4..e024873 --- a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result @@@ -266,8 -266,8 +266,8 @@@ explai select * from t0,t1 where (t0.key1=t1.key1) and (t0.key1=3 or t0.key2<4) and t1.key1=2; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t0 ref i1,i2 i1 4 const 1 Using where + 1 SIMPLE t0 ref i1,i2 i1 4 const 2 Using where -1 SIMPLE t1 ref i1 i1 4 const 2 NULL +1 SIMPLE t1 ref i1 i1 4 const 1 explain select * from t0,t1 where t0.key1 = 5 and (t1.key1 = t0.key1 or t1.key8 = t0.key1); id select_type table type possible_keys key key_len ref rows Extra @@@ -281,12 -281,13 +281,12 @@@ id select_type table type possible_key explain select * from t1 where key1=3 or key2=4 union select * from t1 where key1<4 or key3=5; id select_type table type possible_keys key key_len ref rows Extra - 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where - 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 2 Using sort_union(i1,i3); Using where + 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where + 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 4 Using sort_union(i1,i3); Using where -NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL Using temporary +NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL explain select * from (select * from t1 where key1 = 3 or key2 =3) as Z where key8 >5; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY <derived2> ALL NULL NULL NULL NULL 4 Using where -2 DERIVED t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where +1 SIMPLE t1 index_merge i1,i2,i8 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where create table t3 like t0; insert into t3 select * from t0; alter table t3 add key9 int not null, add index i9(key9); @@@ -1321,16 -1324,16 +1321,16 @@@ id select_type table type possible_key 1 SIMPLE t1 ref key1 key1 4 const ROWS Using where explain select * from t1 where pk1 < 7500 and key1 = 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 PRIMARY 4 NULL ROWS Using where +1 SIMPLE t1 range PRIMARY,key1 key1 8 NULL ROWS Using index condition explain select * from t1 where pktail1ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail1ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail1ok key1,pktail1ok 4,4 NULL 1 Using intersect(key1,pktail1ok); Using where explain select * from t1 where pktail2ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail2ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail2ok key1,pktail2ok 4,4 NULL 1 Using intersect(key1,pktail2ok); Using where explain select * from t1 where (pktail2ok=1 and pk1< 50000) or key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok pktail2ok,key1 8,4 NULL ROWS Using sort_union(pktail2ok,key1); Using where + 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok PRIMARY,key1 4,4 NULL ROWS Using union(PRIMARY,key1); Using where explain select * from t1 where pktail3bad=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref key1,pktail3bad EITHER_KEY 4 const ROWS Using where diff --cc storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index aba14e3,f830440..e22a85f --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@@ -16,8 -12,12 +16,9 @@@ SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WOR select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index dc13bb3,f9a3f81..c6e15dc --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@@ -915,7 -904,8 +918,8 @@@ rocksdb_enable_ttl O rocksdb_enable_ttl_read_filtering ON rocksdb_enable_write_thread_adaptive_yield OFF rocksdb_error_if_exists OFF + rocksdb_error_on_suboptimal_collation ON -rocksdb_flush_log_at_trx_commit 1 +rocksdb_flush_log_at_trx_commit 0 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF @@@ -1457,111 -1446,117 +1465,115 @@@ drop table t0, t1 # show status like 'rocksdb%'; Variable_name Value -rocksdb_rows_deleted # -rocksdb_rows_inserted # -rocksdb_rows_read # -rocksdb_rows_updated # -rocksdb_rows_deleted_blind # -rocksdb_rows_expired # -rocksdb_rows_filtered # -rocksdb_system_rows_deleted # -rocksdb_system_rows_inserted # -rocksdb_system_rows_read # -rocksdb_system_rows_updated # -rocksdb_memtable_total # -rocksdb_memtable_unflushed # -rocksdb_queries_point # -rocksdb_queries_range # -rocksdb_covered_secondary_key_lookups # -rocksdb_block_cache_add # -rocksdb_block_cache_add_failures # -rocksdb_block_cache_bytes_read # -rocksdb_block_cache_bytes_write # -rocksdb_block_cache_data_add # -rocksdb_block_cache_data_bytes_insert # -rocksdb_block_cache_data_hit # -rocksdb_block_cache_data_miss # -rocksdb_block_cache_filter_add # -rocksdb_block_cache_filter_bytes_evict # -rocksdb_block_cache_filter_bytes_insert # -rocksdb_block_cache_filter_hit # -rocksdb_block_cache_filter_miss # -rocksdb_block_cache_hit # -rocksdb_block_cache_index_add # -rocksdb_block_cache_index_bytes_evict # -rocksdb_block_cache_index_bytes_insert # -rocksdb_block_cache_index_hit # -rocksdb_block_cache_index_miss # -rocksdb_block_cache_miss # -rocksdb_block_cachecompressed_hit # -rocksdb_block_cachecompressed_miss # +Rocksdb_rows_deleted # +Rocksdb_rows_inserted # +Rocksdb_rows_read # +Rocksdb_rows_updated # +Rocksdb_rows_deleted_blind # +Rocksdb_rows_expired # +Rocksdb_rows_filtered # +Rocksdb_system_rows_deleted # +Rocksdb_system_rows_inserted # +Rocksdb_system_rows_read # +Rocksdb_system_rows_updated # +Rocksdb_memtable_total # +Rocksdb_memtable_unflushed # +Rocksdb_queries_point # +Rocksdb_queries_range # +Rocksdb_covered_secondary_key_lookups # +Rocksdb_block_cache_add # +Rocksdb_block_cache_add_failures # +Rocksdb_block_cache_bytes_read # +Rocksdb_block_cache_bytes_write # +Rocksdb_block_cache_data_add # +Rocksdb_block_cache_data_bytes_insert # +Rocksdb_block_cache_data_hit # +Rocksdb_block_cache_data_miss # +Rocksdb_block_cache_filter_add # +Rocksdb_block_cache_filter_bytes_evict # +Rocksdb_block_cache_filter_bytes_insert # +Rocksdb_block_cache_filter_hit # +Rocksdb_block_cache_filter_miss # +Rocksdb_block_cache_hit # +Rocksdb_block_cache_index_add # +Rocksdb_block_cache_index_bytes_evict # +Rocksdb_block_cache_index_bytes_insert # +Rocksdb_block_cache_index_hit # +Rocksdb_block_cache_index_miss # +Rocksdb_block_cache_miss # +Rocksdb_block_cachecompressed_hit # +Rocksdb_block_cachecompressed_miss # + rocksdb_bloom_filter_full_positive # + rocksdb_bloom_filter_full_true_positive # -rocksdb_bloom_filter_prefix_checked # -rocksdb_bloom_filter_prefix_useful # -rocksdb_bloom_filter_useful # -rocksdb_bytes_read # -rocksdb_bytes_written # -rocksdb_compact_read_bytes # -rocksdb_compact_write_bytes # -rocksdb_compaction_key_drop_new # -rocksdb_compaction_key_drop_obsolete # -rocksdb_compaction_key_drop_user # -rocksdb_flush_write_bytes # -rocksdb_get_hit_l0 # -rocksdb_get_hit_l1 # -rocksdb_get_hit_l2_and_up # -rocksdb_getupdatessince_calls # -rocksdb_git_date # -rocksdb_git_hash # -rocksdb_iter_bytes_read # +Rocksdb_bloom_filter_prefix_checked # +Rocksdb_bloom_filter_prefix_useful # +Rocksdb_bloom_filter_useful # +Rocksdb_bytes_read # +Rocksdb_bytes_written # +Rocksdb_compact_read_bytes # +Rocksdb_compact_write_bytes # +Rocksdb_compaction_key_drop_new # +Rocksdb_compaction_key_drop_obsolete # +Rocksdb_compaction_key_drop_user # +Rocksdb_flush_write_bytes # +Rocksdb_get_hit_l0 # +Rocksdb_get_hit_l1 # +Rocksdb_get_hit_l2_and_up # +Rocksdb_getupdatessince_calls # +Rocksdb_iter_bytes_read # +Rocksdb_memtable_hit # +Rocksdb_memtable_miss # +Rocksdb_no_file_closes # +Rocksdb_no_file_errors # + rocksdb_manual_compactions_processed # + rocksdb_manual_compactions_running # -rocksdb_memtable_hit # -rocksdb_memtable_miss # -rocksdb_no_file_closes # -rocksdb_no_file_errors # -rocksdb_no_file_opens # -rocksdb_num_iterators # -rocksdb_number_block_not_compressed # -rocksdb_number_db_next # -rocksdb_number_db_next_found # -rocksdb_number_db_prev # -rocksdb_number_db_prev_found # -rocksdb_number_db_seek # -rocksdb_number_db_seek_found # -rocksdb_number_deletes_filtered # -rocksdb_number_keys_read # -rocksdb_number_keys_updated # -rocksdb_number_keys_written # -rocksdb_number_merge_failures # -rocksdb_number_multiget_bytes_read # -rocksdb_number_multiget_get # -rocksdb_number_multiget_keys_read # -rocksdb_number_reseeks_iteration # -rocksdb_number_sst_entry_delete # -rocksdb_number_sst_entry_merge # -rocksdb_number_sst_entry_other # -rocksdb_number_sst_entry_put # -rocksdb_number_sst_entry_singledelete # -rocksdb_number_superversion_acquires # -rocksdb_number_superversion_cleanups # -rocksdb_number_superversion_releases # -rocksdb_row_lock_deadlocks # -rocksdb_row_lock_wait_timeouts # -rocksdb_snapshot_conflict_errors # -rocksdb_stall_l0_file_count_limit_slowdowns # -rocksdb_stall_locked_l0_file_count_limit_slowdowns # -rocksdb_stall_l0_file_count_limit_stops # -rocksdb_stall_locked_l0_file_count_limit_stops # -rocksdb_stall_pending_compaction_limit_stops # -rocksdb_stall_pending_compaction_limit_slowdowns # -rocksdb_stall_memtable_limit_stops # -rocksdb_stall_memtable_limit_slowdowns # -rocksdb_stall_total_stops # -rocksdb_stall_total_slowdowns # -rocksdb_stall_micros # -rocksdb_wal_bytes # -rocksdb_wal_group_syncs # -rocksdb_wal_synced # -rocksdb_write_other # -rocksdb_write_self # -rocksdb_write_timedout # -rocksdb_write_wal # +Rocksdb_no_file_opens # +Rocksdb_num_iterators # +Rocksdb_number_block_not_compressed # +Rocksdb_number_db_next # +Rocksdb_number_db_next_found # +Rocksdb_number_db_prev # +Rocksdb_number_db_prev_found # +Rocksdb_number_db_seek # +Rocksdb_number_db_seek_found # +Rocksdb_number_deletes_filtered # +Rocksdb_number_keys_read # +Rocksdb_number_keys_updated # +Rocksdb_number_keys_written # +Rocksdb_number_merge_failures # +Rocksdb_number_multiget_bytes_read # +Rocksdb_number_multiget_get # +Rocksdb_number_multiget_keys_read # +Rocksdb_number_reseeks_iteration # +Rocksdb_number_sst_entry_delete # +Rocksdb_number_sst_entry_merge # +Rocksdb_number_sst_entry_other # +Rocksdb_number_sst_entry_put # +Rocksdb_number_sst_entry_singledelete # +Rocksdb_number_superversion_acquires # +Rocksdb_number_superversion_cleanups # +Rocksdb_number_superversion_releases # +Rocksdb_row_lock_deadlocks # +Rocksdb_row_lock_wait_timeouts # +Rocksdb_snapshot_conflict_errors # +Rocksdb_stall_l0_file_count_limit_slowdowns # +Rocksdb_stall_locked_l0_file_count_limit_slowdowns # +Rocksdb_stall_l0_file_count_limit_stops # +Rocksdb_stall_locked_l0_file_count_limit_stops # +Rocksdb_stall_pending_compaction_limit_stops # +Rocksdb_stall_pending_compaction_limit_slowdowns # +Rocksdb_stall_memtable_limit_stops # +Rocksdb_stall_memtable_limit_slowdowns # +Rocksdb_stall_total_stops # +Rocksdb_stall_total_slowdowns # +Rocksdb_stall_micros # +Rocksdb_wal_bytes # +Rocksdb_wal_group_syncs # +Rocksdb_wal_synced # +Rocksdb_write_other # +Rocksdb_write_self # +Rocksdb_write_timedout # +Rocksdb_write_wal # select VARIABLE_NAME from INFORMATION_SCHEMA.global_status where VARIABLE_NAME LIKE 'rocksdb%'; VARIABLE_NAME ROCKSDB_ROWS_DELETED @@@ -1617,7 -1614,11 +1631,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES @@@ -1726,7 -1729,11 +1744,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result index 24466d9,0b1d346..b096e42 --- a/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result @@@ -1,11 -1,11 +1,11 @@@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] DROP TABLE IF EXISTS t1; +connection slave; include/stop_slave.inc + create table t1 (a int, b int, primary key (a), unique key (b)) engine=rocksdb; +connection master; - create table t1 (a int) engine=rocksdb; +connection slave; show variables like 'rpl_skip_tx_api'; Variable_name Value rpl_skip_tx_api ON diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test index d715eb7,522ff9f..4f34cbe --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test @@@ -1,6 -1,6 +1,7 @@@ --source include/have_rocksdb.inc --source include/have_debug.inc +--source include/have_partition.inc + --source include/not_valgrind.inc --disable_warnings drop table if exists t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test index 5eac859,be8b9fb..61a10a6 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test @@@ -66,9 -63,7 +66,9 @@@ ALTER TABLE t1 ADD INDEX kb(b), ALGORIT --disable_warnings # now do same index using copy algorithm # hitting max row locks (1M) +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; - --error ER_RDB_STATUS_GENERAL + --error ER_GET_ERRMSG ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; diff --cc storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test index 86a0a7f,238e62a..f4da0b7 --- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test @@@ -117,12 -115,12 +117,35 @@@ INSERT INTO t0 VALUES(0) ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +--echo #--------------------------------------------------------------- +--echo # MDEV-16703 Assertion failed in load_auto_incr_value_from_index +--echo #--------------------------------------------------------------- + +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++ ++--echo #---------------------------------- ++--echo # Issue #792 Crash in autoincrement ++--echo #---------------------------------- ++ ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++ ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; ++ + --echo #---------------------------------- + --echo # Issue #869 Crash in autoincrement + --echo #---------------------------------- + + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test index 9e7c816,c1cdf53..53bfba3 --- a/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test @@@ -8,7 -4,8 +8,10 @@@ let $restart_file= $MYSQLTEST_VARDIR/tm let $error_log= $MYSQLTEST_VARDIR/log/my_restart.err; select variable_name, variable_value from information_schema.global_variables where variable_name="rocksdb_ignore_unknown_options"; ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" +--exec find $MYSQLD_DATADIR/#rocksdb/OPTIONS* | sort -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++ --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --shutdown_server 10 diff --cc storage/rocksdb/mysql-test/rocksdb/t/collation.test index 7152925,38ce1b2..3b808bc --- a/storage/rocksdb/mysql-test/rocksdb/t/collation.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/collation.test @@@ -1,13 -1,7 +1,9 @@@ --source include/have_rocksdb.inc ---source include/have_fullregex.inc +# MariaDB doesn't have server variables to check for GCC version, so the +# following check is commented out: +# --source include/have_fullregex.inc - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - - --disable_warnings - DROP TABLE IF EXISTS t1; - --enable_warnings + call mtr.add_suppression("Invalid pattern"); # ci non-indexed column is allowed CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; diff --cc storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 67dae5d,2ae0b40..09998b9 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@@ -28,10 -17,10 +28,11 @@@ select VALUE into @keysIn from INFORMAT CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --cc storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test index b884738,cf8fb29..f4fdaa7 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test @@@ -1,5 -1,5 +1,6 @@@ --source include/have_rocksdb.inc +--source include/have_partition.inc + --source suite/rocksdb/include/have_write_committed.inc # # RocksDB Storage Engine tests diff --cc storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test index e9dcc60,d9ffb08..9f1e28d --- a/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test @@@ -43,41 -46,47 +45,44 @@@ shutdown_server 10 shutdown_server 10; # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message ---exec echo "Expect errors that we are missing two .frm files" ---let SEARCH_FILE=$LOG ---let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t1 is registered in RocksDB but does not have a .frm file +--echo "Expect errors that we are missing two .frm files" +--let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN=Schema mismatch --source include/search_pattern_in_file.inc + --let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t2 is registered in RocksDB but does not have a .frm file + --source include/search_pattern_in_file.inc # Now shut down again and rename one the .frm file back and make a copy of it --exec echo "wait" >$_expect_file_name shutdown_server 10; - + --remove_file $LOG - # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm ---exec cp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm +--copy_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message for the second case ---exec echo "Expect an error that we have an extra .frm file" +--echo "Expect an error that we have an extra .frm file" - --let SEARCH_PATTERN=Schema mismatch + --let SEARCH_FILE=$LOG + --let SEARCH_PATTERN=Schema mismatch - A .frm file exists for table test.t1_dummy, but that table is not registered in RocksDB --source include/search_pattern_in_file.inc # Shut down an clean up diff --cc storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc index 5e5961f,492b5cd..16ad535 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc @@@ -40,9 -40,9 +40,9 @@@ select * from mysql.slave_gtid_info --write_file $MYSQL_TMP_DIR/truncate_tail_wal.sh #!/bin/bash -F=`ls -t $slave_data_dir/.rocksdb/*.log | head -n 1` +F=`ls -t $slave_data_dir/\#rocksdb/*.log | head -n 1` SIZE=`stat -c %s $F` - NEW_SIZE=`expr $SIZE - 10` + NEW_SIZE=`expr $SIZE - 30` truncate -s $NEW_SIZE $F rc=$? if [[ $rc != 0 ]]; then diff --cc storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test index 39688e6,31b50aa..8d3bab4 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test @@@ -1,7 -1,21 +1,21 @@@ --source include/have_rocksdb.inc + CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO valid_values VALUES(65536); + INSERT INTO valid_values VALUES(1024); + INSERT INTO valid_values VALUES(1*1024*1024); + + CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO invalid_values VALUES('\'aaa\''); + INSERT INTO invalid_values VALUES('\'bbb\''); + INSERT INTO invalid_values VALUES('\'-1\''); + INSERT INTO invalid_values VALUES('\'101\''); + INSERT INTO invalid_values VALUES('\'484436\''); + --let $sys_var=ROCKSDB_BLOCK_CACHE_SIZE - --let $read_only=1 + --let $read_only=0 --let $session=0 ---source ../include/rocksdb_sys_var.inc +--source include/rocksdb_sys_var.inc + DROP TABLE valid_values; + DROP TABLE invalid_values; diff --cc storage/rocksdb/rdb_i_s.cc index de33d69,147ad7f..9192795 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@@ -1757,10 -1869,26 +1906,26 @@@ struct st_maria_plugin rdb_i_s_ddl = nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ + 0, /* flags */ + }; + + struct st_mysql_plugin rdb_i_s_sst_props = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_SST_PROPS", + "Facebook", + "RocksDB SST Properties", + PLUGIN_LICENSE_GPL, + rdb_i_s_sst_props_init, + rdb_i_s_deinit, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; -struct st_mysql_plugin rdb_i_s_index_file_map = { +struct st_maria_plugin rdb_i_s_index_file_map = { MYSQL_INFORMATION_SCHEMA_PLUGIN, &rdb_i_s_info, "ROCKSDB_INDEX_FILE_MAP", diff --cc storage/rocksdb/rdb_i_s.h index d6a48bf,dd9f4a9..f4c4b04 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@@ -21,16 -21,17 +21,17 @@@ namespace myrocks Declare INFORMATION_SCHEMA (I_S) plugins needed by MyRocks storage engine. */ -extern struct st_mysql_plugin rdb_i_s_cfstats; -extern struct st_mysql_plugin rdb_i_s_dbstats; -extern struct st_mysql_plugin rdb_i_s_perf_context; -extern struct st_mysql_plugin rdb_i_s_perf_context_global; -extern struct st_mysql_plugin rdb_i_s_cfoptions; -extern struct st_mysql_plugin rdb_i_s_compact_stats; -extern struct st_mysql_plugin rdb_i_s_global_info; -extern struct st_mysql_plugin rdb_i_s_ddl; -extern struct st_mysql_plugin rdb_i_s_sst_props; -extern struct st_mysql_plugin rdb_i_s_index_file_map; -extern struct st_mysql_plugin rdb_i_s_lock_info; -extern struct st_mysql_plugin rdb_i_s_trx_info; -extern struct st_mysql_plugin rdb_i_s_deadlock_info; +extern struct st_maria_plugin rdb_i_s_cfstats; +extern struct st_maria_plugin rdb_i_s_dbstats; +extern struct st_maria_plugin rdb_i_s_perf_context; +extern struct st_maria_plugin rdb_i_s_perf_context_global; +extern struct st_maria_plugin rdb_i_s_cfoptions; +extern struct st_maria_plugin rdb_i_s_compact_stats; +extern struct st_maria_plugin rdb_i_s_global_info; +extern struct st_maria_plugin rdb_i_s_ddl; ++extern struct st_maria_plugin rdb_i_s_sst_props; +extern struct st_maria_plugin rdb_i_s_index_file_map; +extern struct st_maria_plugin rdb_i_s_lock_info; +extern struct st_maria_plugin rdb_i_s_trx_info; +extern struct st_maria_plugin rdb_i_s_deadlock_info; } // namespace myrocks diff --cc storage/rocksdb/rocksdb index ba295cd,0000000..926f3a7 mode 160000,000000..160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@@ -1,1 -1,0 +1,1 @@@ - Subproject commit ba295cda29daee3ffe58549542804efdfd969784 ++Subproject commit 926f3a78a64b327475ee6c60b6c8ab4f34253204

1 0

[Commits] 25b5345: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge
by psergey＠askmonty.org 28 Aug '18

28 Aug '18

revision-id: 25b5345aff6333f1dfa28f5d95bc363d0bc1f535 parent(s): 55163ba1bdb1a05daadc66c41c959994231b361c faa4d8f8c6c12c09541c4733773ac700c5e643cf committer: Sergei Petrunia branch nick: 10.2-mariarocks timestamp: 2018-08-28 12:49:50 +0300 message: Merge branch 'merge-myrocks' of github.com:MariaDB/mergetrees into bb-10.2-mariarocks-merge Conflicts: storage/rocksdb/CMakeLists.txt storage/rocksdb/ha_rocksdb.cc storage/rocksdb/ha_rocksdb.h storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result storage/rocksdb/mysql-test/rocksdb/r/transaction.result storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result storage/rocksdb/mysql-test/rocksdb/r/validate_datadic.result storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test storage/rocksdb/mysql-test/rocksdb/t/bloomfilter5-master.opt storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test storage/rocksdb/mysql-test/rocksdb/t/collation.test storage/rocksdb/mysql-test/rocksdb/t/information_schema.test storage/rocksdb/mysql-test/rocksdb/t/prefix_extractor_override.test storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test storage/rocksdb/mysql-test/rocksdb/t/rocksdb_cf_options.test storage/rocksdb/mysql-test/rocksdb/t/rocksdb_checksums.test storage/rocksdb/mysql-test/rocksdb/t/transaction.test storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test storage/rocksdb/rdb_i_s.cc storage/rocksdb/rdb_i_s.h storage/rocksdb/CMakeLists.txt | 8 +- storage/rocksdb/ha_rocksdb.cc | 1219 +++++++++++++++----- storage/rocksdb/ha_rocksdb.h | 53 +- storage/rocksdb/ha_rocksdb_proto.h | 7 +- storage/rocksdb/mysql-test/rocksdb/combinations | 6 + .../rocksdb/include/have_write_committed.inc | 3 + .../mysql-test/rocksdb/r/add_index_inplace.result | 24 + .../r/add_index_inplace_sstfilewriter.result | 2 +- .../rocksdb/r/allow_no_primary_key.result | 6 +- .../mysql-test/rocksdb/r/autoinc_debug.result | 40 +- .../mysql-test/rocksdb/r/autoinc_vars.result | 18 + .../mysql-test/rocksdb/r/bloomfilter5.result | 4 +- .../rocksdb/r/bloomfilter_bulk_load.result | 15 + .../mysql-test/rocksdb/r/bulk_load_sk.result | 229 ++++ .../mysql-test/rocksdb/r/cardinality.result | 17 +- .../rocksdb/mysql-test/rocksdb/r/collation.result | 20 +- .../rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result | 21 + .../rocksdb/r/create_no_primary_key_table.result | 38 + .../mysql-test/rocksdb/r/ddl_high_priority.result | 71 +- .../mysql-test/rocksdb/r/deadlock_tracking.result | 57 + .../rocksdb/mysql-test/rocksdb/r/drop_table.result | 11 + .../mysql-test/rocksdb/r/explicit_snapshot.result | 265 +++++ .../mysql-test/rocksdb/r/i_s_deadlock.result | 55 +- .../mysql-test/rocksdb/r/index_file_map.result | 3 + .../rocksdb/r/index_merge_rocksdb2.result | 52 +- .../mysql-test/rocksdb/r/information_schema.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/lock.result | 15 + .../rocksdb/mysql-test/rocksdb/r/mysqldump.result | 5 +- .../r/percona_nonflushing_analyze_debug.result | 19 + .../rocksdb/r/prefix_extractor_override.result | 7 +- .../mysql-test/rocksdb/r/read_only_tx.result | 2 +- .../rocksdb/mysql-test/rocksdb/r/rocksdb.result | 24 +- .../mysql-test/rocksdb/r/rocksdb_checksums.result | 6 +- .../rocksdb/r/rocksdb_deadlock_detect_rc.result | 16 +- .../rocksdb/r/rocksdb_deadlock_detect_rr.result | 16 +- .../rocksdb/r/secondary_key_update_lock.result | 11 + .../mysql-test/rocksdb/r/show_engine.result | 53 + .../mysql-test/rocksdb/r/transaction.result | 17 + .../mysql-test/rocksdb/r/trx_info_rpl.result | 2 +- storage/rocksdb/mysql-test/rocksdb/r/update.result | 8 + .../mysql-test/rocksdb/r/use_direct_reads.result | 18 + .../mysql-test/rocksdb/t/add_index_inplace.test | 18 + .../rocksdb/t/add_index_inplace_crash.test | 1 + .../rocksdb/t/add_index_inplace_sstfilewriter.test | 2 +- .../mysql-test/rocksdb/t/autoinc_debug.test | 17 +- .../rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 23 + .../mysql-test/rocksdb/t/bloomfilter5-master.opt | 4 +- .../rocksdb/t/bloomfilter_bulk_load-master.opt | 2 + .../rocksdb/t/bloomfilter_bulk_load.test | 35 + .../rocksdb/mysql-test/rocksdb/t/bulk_load_sk.test | 119 ++ .../rocksdb/mysql-test/rocksdb/t/cardinality.test | 21 +- .../rocksdb/t/check_ignore_unknown_options.test | 3 + .../rocksdb/mysql-test/rocksdb/t/collation.test | 36 +- .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.cnf | 4 + .../rocksdb/mysql-test/rocksdb/t/com_rpc_tx.test | 87 ++ .../t/create_no_primary_key_table-master.opt | 1 + .../rocksdb/t/create_no_primary_key_table.test | 44 + .../mysql-test/rocksdb/t/deadlock_tracking.test | 76 +- .../rocksdb/mysql-test/rocksdb/t/drop_table.test | 27 + .../rocksdb/t/explicit_snapshot-master.opt | 1 + .../mysql-test/rocksdb/t/explicit_snapshot.test | 263 +++++ .../rocksdb/mysql-test/rocksdb/t/i_s_deadlock.test | 14 +- .../mysql-test/rocksdb/t/index_file_map.test | 3 + .../rocksdb/t/index_merge_rocksdb2-master.opt | 2 +- .../mysql-test/rocksdb/t/information_schema.test | 1 + .../rocksdb/t/insert_optimized_config-master.opt | 8 - .../rocksdb/t/insert_optimized_config.test | 10 + storage/rocksdb/mysql-test/rocksdb/t/lock.test | 22 + .../t/percona_nonflushing_analyze_debug.test | 11 + .../rocksdb/t/prefix_extractor_override.test | 14 +- storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test | 3 +- .../mysql-test/rocksdb/t/rocksdb_checksums.test | 10 +- .../rocksdb/t/rocksdb_deadlock_detect.inc | 35 +- .../rocksdb/t/secondary_key_update_lock.test | 26 + .../mysql-test/rocksdb/t/set_checkpoint.inc | 2 +- .../rocksdb/mysql-test/rocksdb/t/show_engine.test | 13 + .../rocksdb/mysql-test/rocksdb/t/transaction.test | 23 + .../rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test | 4 +- storage/rocksdb/mysql-test/rocksdb/t/update.test | 10 + .../mysql-test/rocksdb/t/use_direct_reads.test | 37 + .../mysql-test/rocksdb/t/validate_datadic.test | 16 +- .../rocksdb/mysql-test/rocksdb_rpl/combinations | 8 +- .../r/rpl_missing_columns_sk_update.result | 62 + .../rpl_mts_dependency_unique_key_conflicts.result | 44 + .../rocksdb_rpl/r/rpl_rocksdb_snapshot.result | 4 +- .../t/rpl_gtid_crash_safe_wal_corrupt.inc | 2 +- .../t/rpl_missing_columns_sk_update.cnf | 13 + .../t/rpl_missing_columns_sk_update.test | 69 ++ .../t/rpl_mts_dependency_unique_key_conflicts.test | 64 + .../t/rpl_rocksdb_2pc_crash_recover.test | 1 + .../rocksdb/mysql-test/rocksdb_stress/combinations | 5 + .../r/rocksdb_block_cache_size_basic.result | 84 +- .../r/rocksdb_bulk_load_allow_sk_basic.result | 100 ++ ...sdb_commit_time_batch_for_recovery_basic.result | 121 ++ .../r/rocksdb_create_checkpoint_basic.result | 2 +- ...ksdb_debug_manual_compaction_delay_basic.result | 46 + ...ksdb_error_on_suboptimal_collation_basic.result | 7 + .../rocksdb_manual_compaction_threads_basic.result | 93 ++ .../r/rocksdb_max_manifest_file_size_basic.result | 2 +- .../r/rocksdb_max_manual_compactions_basic.result | 57 + .../r/rocksdb_max_row_locks_basic.result | 21 + .../r/rocksdb_stats_recalc_rate_basic.result | 53 + .../r/rocksdb_write_batch_max_bytes_basic.result | 4 +- .../r/rocksdb_write_policy_basic.result | 15 + .../t/rocksdb_block_cache_size_basic.test | 16 +- .../t/rocksdb_bulk_load_allow_sk_basic.test | 18 + ...cksdb_commit_time_batch_for_recovery_basic.test | 20 + .../t/rocksdb_create_checkpoint_basic.test | 2 +- ...ocksdb_debug_manual_compaction_delay_basic.test | 16 + ...ocksdb_error_on_suboptimal_collation_basic.test | 6 + .../t/rocksdb_manual_compaction_threads_basic.test | 17 + .../t/rocksdb_max_manual_compactions_basic.test | 17 + .../t/rocksdb_max_row_locks_basic.test | 1 + .../t/rocksdb_stats_recalc_rate_basic.test | 17 + .../t/rocksdb_write_batch_max_bytes_basic.test | 4 +- .../t/rocksdb_write_policy_basic.test | 17 + storage/rocksdb/properties_collector.cc | 61 +- storage/rocksdb/properties_collector.h | 3 +- storage/rocksdb/rdb_cf_options.cc | 2 - storage/rocksdb/rdb_comparator.h | 46 +- storage/rocksdb/rdb_datadic.cc | 6 +- storage/rocksdb/rdb_datadic.h | 5 +- storage/rocksdb/rdb_i_s.cc | 167 ++- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_index_merge.cc | 5 + storage/rocksdb/rdb_perf_context.cc | 2 - storage/rocksdb/rdb_psi.cc | 17 +- storage/rocksdb/rdb_psi.h | 10 +- storage/rocksdb/rdb_sst_info.cc | 4 +- storage/rocksdb/rdb_threads.h | 31 +- storage/rocksdb/rdb_utils.cc | 23 - storage/rocksdb/rocksdb | 2 +- 132 files changed, 4286 insertions(+), 639 deletions(-) diff --cc storage/rocksdb/CMakeLists.txt index 2a4ea4d,54b0b1f..2689dc7 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@@ -130,55 -53,64 +130,58 @@@ if(HAVE_THREAD_LOCAL ADD_DEFINITIONS(-DROCKSDB_SUPPORT_THREAD_LOCAL) endif() -SET(ROCKSDB_SOURCES - ha_rocksdb.cc ha_rocksdb.h ha_rocksdb_proto.h +INCLUDE(build_rocksdb.cmake) + +ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib + ha_rocksdb_proto.h logger.h rdb_comparator.h - rdb_datadic.cc rdb_datadic.h - rdb_cf_options.cc rdb_cf_options.h - rdb_cf_manager.cc rdb_cf_manager.h - properties_collector.cc properties_collector.h - event_listener.cc event_listener.h - rdb_i_s.cc rdb_i_s.h - rdb_index_merge.cc rdb_index_merge.h - rdb_io_watchdog.cc rdb_io_watchdog.h - rdb_perf_context.cc rdb_perf_context.h - rdb_mutex_wrapper.cc rdb_mutex_wrapper.h - rdb_psi.h rdb_psi.cc - rdb_sst_info.cc rdb_sst_info.h - rdb_utils.cc rdb_utils.h rdb_buff.h - rdb_threads.cc rdb_threads.h - ${ROCKSDB_LIB_SOURCES} + rdb_cf_options.cc + rdb_cf_options.h + event_listener.cc + event_listener.h + rdb_perf_context.cc + rdb_perf_context.h + rdb_buff.h + rdb_mariadb_port.h ) -IF(WITH_FB_TSAN) - SET(PIC_EXT "_pic") -ELSE() - SET(PIC_EXT "") -ENDIF() +ADD_DEPENDENCIES(rocksdb_aux_lib GenError) -SET(rocksdb_static_libs ) -IF (WITH_SNAPPY) - FIND_LIBRARY(SNAPPY_LIBRARY - NAMES libsnappy${PIC_EXT}.a snappy - HINTS ${WITH_SNAPPY}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${SNAPPY_LIBRARY}) - ADD_DEFINITIONS(-DSNAPPY) -ENDIF() +# MARIAROCKS-TODO: how to properly depend on -lrt ? +TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY}) +if (UNIX AND NOT APPLE) + TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt) +endif() -IF (WITH_LZ4) +TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib) + FIND_LIBRARY(LZ4_LIBRARY + NAMES liblz4${PIC_EXT}.a lz4 + HINTS ${WITH_LZ4}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${LZ4_LIBRARY}) - ADD_DEFINITIONS(-DLZ4) -ENDIF() -IF (WITH_BZ2) - FIND_LIBRARY(BZ2_LIBRARY - NAMES libbz2${PIC_EXT}.a bz2 - HINTS ${WITH_BZ2}/lib) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${BZ2_LIBRARY}) - ADD_DEFINITIONS(-DBZIP2) +IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + + # MARIAROCKS_NOT_YET: Add -frtti flag when compiling RocksDB files. + # TODO: is this the right way to do this? + # - SQL layer and storage/rocksdb/*.cc are compiled with -fnortti + # - RocksDB files are compiled with "-fnortti ... -frtti" + # - This causes RocksDB headers to be compiled with different settings: + # = with RTTI when compiling RocksDB + # = without RTTI when compiling storage/rocksdb/*.cc + # + # (facebook/mysql-5.6 just compiles everything without -f*rtti, which means + # everything is compiled with -frtti) + # + # (also had to add -frtti above, because something that event_listener.cc + # includes requires it. So, now everything in MariaRocks is compiled with + # -frtti) + set_source_files_properties(event_listener.cc rdb_cf_options.cc rdb_sst_info.cc + PROPERTIES COMPILE_FLAGS -frtti) ENDIF() -IF (WITH_ZSTD) - SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZSTD_LIBRARY}) - ADD_DEFINITIONS(-DZSTD) +CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU) +IF(HAVE_SCHED_GETCPU) + ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1) ENDIF() IF (WITH_TBB) diff --cc storage/rocksdb/ha_rocksdb.cc index adef71e,c7d7cc0..36054ac --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@@ -506,77 -529,20 +545,85 @@@ static my_bool rocksdb_enable_bulk_load static my_bool rocksdb_print_snapshot_conflict_queries = 0; static my_bool rocksdb_large_prefix = 0; static my_bool rocksdb_allow_to_start_after_corruption = 0; +static char* rocksdb_git_hash; + +char *compression_types_val= + const_cast<char*>(get_rocksdb_supported_compression_types()); + static uint64_t rocksdb_write_policy = + rocksdb::TxnDBWritePolicy::WRITE_COMMITTED; + static my_bool rocksdb_error_on_suboptimal_collation = 1; + static uint32_t rocksdb_stats_recalc_rate = 0; + static uint32_t rocksdb_debug_manual_compaction_delay = 0; + static uint32_t rocksdb_max_manual_compactions = 0; std::atomic<uint64_t> rocksdb_row_lock_deadlocks(0); std::atomic<uint64_t> rocksdb_row_lock_wait_timeouts(0); std::atomic<uint64_t> rocksdb_snapshot_conflict_errors(0); std::atomic<uint64_t> rocksdb_wal_group_syncs(0); + std::atomic<uint64_t> rocksdb_manual_compactions_processed(0); + std::atomic<uint64_t> rocksdb_manual_compactions_running(0); + + +/* + Remove directory with files in it. + Used to remove checkpoint created by mariabackup. +*/ +#ifdef _WIN32 +#include <direct.h> /* unlink*/ +#ifndef F_OK +#define F_OK 0 +#endif +#endif + +static int rmdir_force(const char *dir) { + if (access(dir, F_OK)) + return true; + + char path[FN_REFLEN]; + char sep[] = {FN_LIBCHAR, 0}; + int err = 0; + + MY_DIR *dir_info = my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i = 0; i < dir_info->number_of_files; i++) { + FILEINFO *file = dir_info->dir_entry + i; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + err = my_delete(path, 0); + + if (err) { + break; + } + } + + my_dirend(dir_info); + + if (!err) + err = rmdir(dir); + + return (err == 0) ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; +} + + +static void rocksdb_remove_mariabackup_checkpoint( + my_core::THD *const, + struct st_mysql_sys_var *const , + void *const var_ptr, const void *const) { + std::string mariabackup_checkpoint_dir(rocksdb_datadir); + + mariabackup_checkpoint_dir.append("/mariabackup-checkpoint"); + + if (unlink(mariabackup_checkpoint_dir.c_str()) == 0) + return; + + rmdir_force(mariabackup_checkpoint_dir.c_str()); +} + + static std::unique_ptr<rocksdb::DBOptions> rdb_init_rocksdb_db_options(void) { auto o = std::unique_ptr<rocksdb::DBOptions>(new rocksdb::DBOptions()); @@@ -1077,9 -1076,11 +1165,11 @@@ static MYSQL_SYSVAR_INT(table_cache_num "DBOptions::table_cache_numshardbits for RocksDB", nullptr, nullptr, rocksdb_db_options->table_cache_numshardbits, - /* min */ 0, /* max */ INT_MAX, 0); + // LRUCache limits this to 19 bits, anything greater + // fails to create a cache and returns a nullptr + /* min */ 0, /* max */ 19, 0); -static MYSQL_SYSVAR_ULONG(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, +static MYSQL_SYSVAR_UINT64_T(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "DBOptions::WAL_ttl_seconds for RocksDB", nullptr, nullptr, rocksdb_db_options->WAL_ttl_seconds, @@@ -1753,7 -1772,11 +1873,12 @@@ static struct st_mysql_sys_var *rocksdb MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(allow_to_start_after_corruption), + MYSQL_SYSVAR(git_hash), + MYSQL_SYSVAR(error_on_suboptimal_collation), + MYSQL_SYSVAR(stats_recalc_rate), + MYSQL_SYSVAR(debug_manual_compaction_delay), + MYSQL_SYSVAR(max_manual_compactions), + MYSQL_SYSVAR(manual_compaction_threads), nullptr}; static rocksdb::WriteOptions @@@ -1897,9 -1941,7 +2064,9 @@@ private ulonglong m_writes_at_last_savepoint; protected: - THD *m_thd = nullptr; + - rocksdb::ReadOptions m_read_opts; ++protected: + THD *m_thd = nullptr; static std::multiset<Rdb_transaction *> s_tx_list; static mysql_mutex_t s_tx_list_mutex; @@@ -1956,14 -1990,20 +2123,23 @@@ protected return s; } + protected: + /* + The following two are helper functions to be overloaded by child classes. + They should provide RocksDB's savepoint semantics. + */ + virtual void do_set_savepoint() = 0; + virtual void do_rollback_to_savepoint() = 0; + public: + rocksdb::ReadOptions m_read_opts; const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; +#ifdef MARIAROCKS_NOT_YET + // TODO: MariaDB probably doesn't need these at all: const char *m_mysql_gtid; const char *m_mysql_max_gtid; +#endif String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; @@@ -2946,6 -2991,6 +3141,7 @@@ error m_batch->RollbackToSavePoint(); } ++ public: bool is_writebatch_trx() const override { return true; } @@@ -3563,7 -3494,8 +3766,8 @@@ static int rocksdb_commit(handlerton* h /* We get here when committing a statement within a transaction. */ - tx->set_tx_failed(false); + tx->make_stmt_savepoint_permanent(); + tx->make_stmt_savepoint_permanent(); } if (my_core::thd_tx_isolation(thd) <= ISO_READ_COMMITTED) { @@@ -4119,9 -4052,24 +4326,25 @@@ static bool rocksdb_show_status(handler str, stat_print); } } - #endif + /* Explicit snapshot information */ + str.clear(); + { + std::lock_guard<std::mutex> lock(explicit_snapshot_mutex); + for (const auto &elem : explicit_snapshots) { + const auto &ss = elem.second.lock(); + DBUG_ASSERT(ss != nullptr); + const auto &info = ss->ss_info; + str += "\nSnapshot ID: " + std::to_string(info.snapshot_id) + + "\nBinlog File: " + info.binlog_file + + "\nBinlog Pos: " + std::to_string(info.binlog_pos) + + "\nGtid Executed: " + info.gtid_executed + "\n"; + } + } + if (!str.empty()) { + res |= print_stats(thd, "EXPLICIT_SNAPSHOTS", "rocksdb", str, stat_print); + } +#ifdef MARIAROCKS_NOT_YET } else if (stat_type == HA_ENGINE_TRX) { /* Handle the SHOW ENGINE ROCKSDB TRANSACTION STATUS command */ res |= rocksdb_show_snapshot_status(hton, thd, stat_print); @@@ -4141,8 -4089,48 +4364,50 @@@ static inline void rocksdb_register_tx( } } +static const char *ha_rocksdb_exts[] = {NullS}; + + static bool rocksdb_explicit_snapshot( + handlerton *const /* hton */, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle */ + snapshot_info_st *ss_info) /*!< out: Snapshot information */ + { + switch (ss_info->op) { + case snapshot_operation::SNAPSHOT_CREATE: { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + auto s = Rdb_explicit_snapshot::create(ss_info, rdb, rdb->GetSnapshot()); + if (mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + thd->set_explicit_snapshot(s); + return s == nullptr; + } + case snapshot_operation::SNAPSHOT_ATTACH: { + auto s = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!s) { + return true; + } + *ss_info = s->ss_info; + thd->set_explicit_snapshot(s); + return false; + } + case snapshot_operation::SNAPSHOT_RELEASE: { + if (!thd->get_explicit_snapshot()) { + return true; + } + *ss_info = thd->get_explicit_snapshot()->ss_info; + thd->set_explicit_snapshot(nullptr); + return false; + } + default: + DBUG_ASSERT(false); + return true; + } + return true; + } + /* Supporting START TRANSACTION WITH CONSISTENT [ROCKSDB] SNAPSHOT @@@ -4176,16 -4166,14 +4443,27 @@@ static int rocksdb_start_tx_and_assign_ my_error(ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT, MYF(0)); return HA_EXIT_FAILURE; } + ++#ifdef MARIADB_NOT_YET + if (ss_info) { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } else { + return HA_EXIT_FAILURE; + } - } ++#endif ++ + /* + MariaDB: there is no need to call mysql_bin_log_lock_commits and then + unlock back. + SQL layer calls start_consistent_snapshot() for all engines, including the + binlog under LOCK_commit_ordered mutex. ++ + The mutex prevents binlog commits from happening (right?) while the storage + engine(s) allocate read snapshots. That way, each storage engine is - synchronized with current binlog position. ++ synchronized with current binlog position. + */ + mysql_mutex_assert_owner(&LOCK_commit_ordered); Rdb_transaction *const tx = get_or_create_tx(thd); Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); @@@ -4195,6 -4183,10 +4473,11 @@@ rocksdb_register_tx(hton, thd, tx); tx->acquire_snapshot(true); ++#ifdef MARIADB_NOT_YET + if (ss_info) { + mysql_bin_log_unlock_commits(ss_info); + } - ++#endif return HA_EXIT_SUCCESS; } @@@ -4445,6 -4502,9 +4811,8 @@@ static int rocksdb_init_func(void *cons mysql_mutex_init(rdb_sysvars_psi_mutex_key, &rdb_sysvars_mutex, MY_MUTEX_INIT_FAST); + mysql_mutex_init(rdb_block_cache_resize_mutex_key, + &rdb_block_cache_resize_mutex, MY_MUTEX_INIT_FAST); - rdb_open_tables.init_hash(); Rdb_transaction::init_mutex(); rocksdb_hton->state = SHOW_OPTION_YES; @@@ -4457,16 -4514,14 +4825,18 @@@ rocksdb_hton->commit_by_xid = rocksdb_commit_by_xid; rocksdb_hton->rollback_by_xid = rocksdb_rollback_by_xid; rocksdb_hton->recover = rocksdb_recover; + + rocksdb_hton->commit_ordered= rocksdb_commit_ordered; rocksdb_hton->commit = rocksdb_commit; + + rocksdb_hton->commit_checkpoint_request= rocksdb_checkpoint_request; + rocksdb_hton->rollback = rocksdb_rollback; - rocksdb_hton->db_type = DB_TYPE_ROCKSDB; rocksdb_hton->show_status = rocksdb_show_status; + rocksdb_hton->explicit_snapshot = rocksdb_explicit_snapshot; rocksdb_hton->start_consistent_snapshot = rocksdb_start_tx_and_assign_read_view; + rocksdb_hton->start_shared_snapshot = rocksdb_start_tx_with_shared_read_view; rocksdb_hton->savepoint_set = rocksdb_savepoint; rocksdb_hton->savepoint_rollback = rocksdb_rollback_to_savepoint; rocksdb_hton->savepoint_rollback_can_release_mdl = @@@ -4710,9 -4791,10 +5111,9 @@@ } cf_manager.init(std::move(cf_options_map), &cf_handles); - if (dict_manager.init(rdb->GetBaseDB(), &cf_manager)) { + if (dict_manager.init(rdb, &cf_manager)) { // NO_LINT_DEBUG sql_print_error("RocksDB: Failed to initialize data dictionary."); - rdb_open_tables.free_hash(); DBUG_RETURN(HA_EXIT_FAILURE); } @@@ -4877,66 -4968,28 +5293,77 @@@ static int rocksdb_done_func(void *cons sql_print_error("RocksDB: Couldn't stop the index thread: (errno=%d)", err); } + // signal the manual compaction thread to stop + rdb_mc_thread.signal(true); + // Wait for the manual compaction thread to finish. + err = rdb_mc_thread.join(); + if (err != 0) { + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Couldn't stop the manual compaction thread: (errno=%d)", err); + } + - if (rdb_open_tables.m_hash.records) { + if (rdb_open_tables.m_hash.size()) { // Looks like we are getting unloaded and yet we have some open tables // left behind. error = 1; } - rdb_open_tables.free_hash(); + /* + MariaDB: When the plugin is unloaded with UNINSTALL SONAME command, some + connections may still have Rdb_transaction objects. + + These objects are not genuine transactions (as SQL layer makes sure that + a plugin that is being unloaded has no open tables), they are empty + Rdb_transaction objects that were left there to save on object + creation/deletion. + + Go through the list and delete them. + */ + { + class Rdb_trx_deleter: public Rdb_tx_list_walker { + public: + std::set<Rdb_transaction*> rdb_trxs; + + void process_tran(const Rdb_transaction *const tx) override { + /* + Check if the transaction is really empty. We only check + non-WriteBatch-based transactions, because there is no easy way to + check WriteBatch-based transactions. + */ + if (!tx->is_writebatch_trx()) { + const auto tx_impl = static_cast<const Rdb_transaction_impl *>(tx); + DBUG_ASSERT(tx_impl); + if (tx_impl->get_rdb_trx()) + DBUG_ASSERT(0); + } + rdb_trxs.insert((Rdb_transaction*)tx); + }; + } deleter; + + Rdb_transaction::walk_tx_list(&deleter); + + for (std::set<Rdb_transaction*>::iterator it= deleter.rdb_trxs.begin(); + it != deleter.rdb_trxs.end(); + ++it) + { + // When a transaction is deleted, it removes itself from s_tx_list. + delete *it; + } + } + + /* + destructors for static objects can be called at _exit(), + but we want to free the memory at dlclose() + */ + rdb_open_tables.m_hash.~Rdb_table_set(); mysql_mutex_destroy(&rdb_open_tables.m_mutex); mysql_mutex_destroy(&rdb_sysvars_mutex); + mysql_mutex_destroy(&rdb_block_cache_resize_mutex); + delete rdb_collation_exceptions; + mysql_mutex_destroy(&rdb_collation_data_mutex); mysql_mutex_destroy(&rdb_mem_cmp_space_mutex); @@@ -6307,14 -6348,13 +6729,14 @@@ void ha_rocksdb::free_key_buffers() my_free(m_dup_sk_packed_tuple_old); m_dup_sk_packed_tuple_old = nullptr; - my_free(m_eq_cond_upper_bound); - m_eq_cond_upper_bound = nullptr; + my_free(m_scan_it_lower_bound); + m_scan_it_lower_bound = nullptr; - my_free(m_eq_cond_lower_bound); - m_eq_cond_lower_bound = nullptr; + my_free(m_scan_it_upper_bound); + m_scan_it_upper_bound = nullptr; } +#ifdef MARIAROCKS_NOT_YET void ha_rocksdb::set_use_read_free_rpl(const char *const whitelist) { DBUG_ASSERT(whitelist != nullptr); @@@ -6796,12 -6766,12 +7216,10 @@@ int ha_rocksdb::create_cfs std::array<struct key_def_cf_info, MAX_INDEXES + 1> *const cfs) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); char tablename_sys[NAME_LEN + 1]; - - my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), - tablename_sys, sizeof(tablename_sys)); + bool tsys_set= false; /* The first loop checks the index parameters and creates @@@ -6810,40 -6780,37 +7228,49 @@@ for (uint i = 0; i < tbl_def_arg->m_key_count; i++) { rocksdb::ColumnFamilyHandle *cf_handle; - if (rocksdb_strict_collation_check && - !is_hidden_pk(i, table_arg, tbl_def_arg) && + if (!is_hidden_pk(i, table_arg, tbl_def_arg) && tbl_def_arg->base_tablename().find(tmp_file_prefix) != 0) { - for (uint part = 0; part < table_arg->key_info[i].actual_key_parts; - part++) { - if (!rdb_is_index_collation_supported( + if (!tsys_set) + { + tsys_set= true; + my_core::filename_to_tablename(tbl_def_arg->base_tablename().c_str(), + tablename_sys, sizeof(tablename_sys)); + } + + for (uint part = 0; part < table_arg->key_info[i].ext_key_parts; + part++) + { + /* MariaDB: disallow NOPAD collations */ + if (rdb_field_uses_nopad_collation( + table_arg->key_info[i].key_part[part].field)) + { + my_error(ER_MYROCKS_CANT_NOPAD_COLLATION, MYF(0)); + DBUG_RETURN(HA_EXIT_FAILURE); + } + + if (rocksdb_strict_collation_check && + !rdb_is_index_collation_supported( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - std::string collation_err; - for (const auto &coll : RDB_INDEX_COLLATIONS) { - if (collation_err != "") { - collation_err += ", "; - } - collation_err += coll->name; - } - if (rocksdb_error_on_suboptimal_collation) { - my_error(ER_UNSUPPORTED_COLLATION, MYF(0), - tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name, - collation_err.c_str()); - DBUG_RETURN(HA_EXIT_FAILURE); + char buf[1024]; + my_snprintf(buf, sizeof(buf), + "Indexed column %s.%s uses a collation that does not " + "allow index-only access in secondary key and has " + "reduced disk space efficiency in primary key.", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name); + + my_error(ER_INTERNAL_ERROR, MYF(ME_JUST_WARNING), buf); + } else { + push_warning_printf( + ha_thd(), Sql_condition::WARN_LEVEL_WARN, ER_WRONG_ARGUMENTS, + "Unsupported collation on string indexed column %s.%s Use " + "binary collation (%s).", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name, + collation_err.c_str()); + } } } } @@@ -7212,10 -7163,7 +7623,8 @@@ int ha_rocksdb::create_key_def(const TA int rdb_normalize_tablename(const std::string &tablename, std::string *const strbuf) { - DBUG_ASSERT(strbuf != nullptr); - - if (tablename.size() < 2 || tablename[0] != '.' || tablename[1] != '/') { + if (tablename.size() < 2 || tablename[0] != '.' || + (tablename[1] != FN_LIBCHAR && tablename[1] != FN_LIBCHAR2)) { DBUG_ASSERT(0); // We were not passed table name? return HA_ERR_ROCKSDB_INVALID_TABLE; } @@@ -7875,11 -7809,10 +8278,10 @@@ ulong ha_rocksdb::index_flags(uint inx pair for. */ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { - DBUG_ASSERT(buf != nullptr); DBUG_ASSERT(table != nullptr); - +#ifdef MARIAROCKS_NOT_YET stats.rows_requested++; - +#endif /* Use STATUS_NOT_FOUND when record not found or some error occurred */ table->status = STATUS_NOT_FOUND; @@@ -9798,34 -9666,23 +10187,28 @@@ int ha_rocksdb::update_write_row(const If the index was reverse order, upper bound would be 0x0000b3eb003f65c5e78857, and lower bound would be 0x0000b3eb003f65c5e78859. These cover given eq condition range. + + @param lower_bound_buf IN Buffer for lower bound + @param upper_bound_buf IN Buffer for upper bound + + @param outer_u */ - void ha_rocksdb::setup_iterator_bounds(const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { - uint eq_cond_len = eq_cond.size(); - memcpy(upper_bound_buf, eq_cond.data(), eq_cond_len); - kd.successor(upper_bound_buf, eq_cond_len); - memcpy(lower_bound_buf, eq_cond.data(), eq_cond_len); - kd.predecessor(lower_bound_buf, eq_cond_len); + void ha_rocksdb::setup_iterator_bounds( + const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, size_t bound_len, + uchar *const lower_bound, uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, rocksdb::Slice *upper_bound_slice) { + uint min_len = std::min(eq_cond.size(), bound_len); + memcpy(upper_bound, eq_cond.data(), min_len); + kd.successor(upper_bound, min_len); + memcpy(lower_bound, eq_cond.data(), min_len); + kd.predecessor(lower_bound, min_len); if (kd.m_is_reverse_cf) { - *out_upper_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); } else { - *out_upper_bound = - rocksdb::Slice((const char *)upper_bound_buf, eq_cond_len); - *out_lower_bound = - rocksdb::Slice((const char *)lower_bound_buf, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); } } @@@ -9845,11 -9701,10 +10227,11 @@@ void ha_rocksdb::setup_scan_iterator(co bool skip_bloom = true; const rocksdb::Slice eq_cond(slice->data(), eq_cond_len); - if (check_bloom_and_set_bounds(ha_thd(), kd, eq_cond, use_all_keys, - m_eq_cond_lower_bound, - m_eq_cond_upper_bound, - &m_eq_cond_lower_bound_slice, + if (check_bloom_and_set_bounds( + ha_thd(), kd, eq_cond, use_all_keys, eq_cond_len, + m_scan_it_lower_bound, m_scan_it_upper_bound, + &m_scan_it_lower_bound_slice, &m_scan_it_upper_bound_slice)) { + &m_eq_cond_upper_bound_slice)) { skip_bloom = false; } @@@ -9955,9 -9810,10 +10337,9 @@@ int ha_rocksdb::rnd_next(uchar *const b DBUG_ENTER_FUNC(); int rc; - ha_statistic_increment(&SSV::ha_read_rnd_next_count); for (;;) { rc = rnd_next_with_direction(buf, true); - if (rc != HA_ERR_LOCK_DEADLOCK || !m_rnd_scan_is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !m_rnd_scan_is_new_snapshot) break; /* exit the loop */ // release the snapshot and iterator and then regenerate them @@@ -10362,12 -10214,23 +10744,23 @@@ int ha_rocksdb::info(uint flag) continue; } KEY *const k = &table->key_info[i]; - for (uint j = 0; j < k->actual_key_parts; j++) { + for (uint j = 0; j < k->ext_key_parts; j++) { const Rdb_index_stats &k_stats = m_key_descr_arr[i]->m_stats; - uint x = k_stats.m_distinct_keys_per_prefix.size() > j && - k_stats.m_distinct_keys_per_prefix[j] > 0 - ? k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j] - : 0; + uint x; + + if (k_stats.m_distinct_keys_per_prefix.size() > j && + k_stats.m_distinct_keys_per_prefix[j] > 0) { + x = k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j]; + /* + If the number of rows is less than the number of prefixes (due to + sampling), the average number of rows with the same prefix is 1. + */ + if (x == 0) { + x = 1; + } + } else { + x = 0; + } if (x > stats.records) x = stats.records; if ((x == 0 && rocksdb_debug_optimizer_no_zero_cardinality) || @@@ -12807,24 -12661,198 +13225,195 @@@ void Rdb_background_thread::run() rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); } } - + // Recalculate statistics for indexes. + if (rocksdb_stats_recalc_rate) { + std::unordered_map<GL_INDEX_ID, std::shared_ptr<const Rdb_key_def>> + to_recalc; + + if (rdb_indexes_to_recalc.empty()) { + struct Rdb_index_collector : public Rdb_tables_scanner { + int add_table(Rdb_tbl_def *tdef) override { + for (uint i = 0; i < tdef->m_key_count; i++) { + rdb_indexes_to_recalc.push_back( + tdef->m_key_descr_arr[i]->get_gl_index_id()); + } + return HA_EXIT_SUCCESS; + } + } collector; + ddl_manager.scan_for_tables(&collector); + } + + while (to_recalc.size() < rocksdb_stats_recalc_rate && + !rdb_indexes_to_recalc.empty()) { + const auto index_id = rdb_indexes_to_recalc.back(); + rdb_indexes_to_recalc.pop_back(); + + std::shared_ptr<const Rdb_key_def> keydef = + ddl_manager.safe_find(index_id); + + if (keydef) { + to_recalc.insert(std::make_pair(keydef->get_gl_index_id(), keydef)); + } + } + + if (!to_recalc.empty()) { + calculate_stats(to_recalc, false); + } + } + - // Set the next timestamp for mysql_cond_timedwait() (which ends up calling - // pthread_cond_timedwait()) to wait on. - ts_next_sync.tv_sec = ts.tv_sec + WAKE_UP_INTERVAL; } // save remaining stats which might've left unsaved ddl_manager.persist_stats(); } - bool ha_rocksdb::check_bloom_and_set_bounds(THD *thd, const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - const bool use_all_keys, - uchar *lower_bound_buf, - uchar *upper_bound_buf, - rocksdb::Slice *out_lower_bound, - rocksdb::Slice *out_upper_bound) { + /* + A background thread to handle manual compactions, + except for dropping indexes/tables. Every second, it checks + pending manual compactions, and it calls CompactRange if there is. + */ + void Rdb_manual_compaction_thread::run() { + mysql_mutex_init(0, &m_mc_mutex, MY_MUTEX_INIT_FAST); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + for (;;) { + if (m_stop) { + break; + } + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 1; + + const auto ret MY_ATTRIBUTE((__unused__)) = + mysql_cond_timedwait(&m_signal_cond, &m_signal_mutex, &ts); + if (m_stop) { + break; + } + // make sure, no program error is returned + DBUG_ASSERT(ret == 0 || ret == ETIMEDOUT); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + // Grab the first item and proceed, if not empty. + if (m_requests.empty()) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + continue; + } + Manual_compaction_request &mcr = m_requests.begin()->second; + DBUG_ASSERT(mcr.cf != nullptr); + DBUG_ASSERT(mcr.state == Manual_compaction_request::INITED); + mcr.state = Manual_compaction_request::RUNNING; + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + + DBUG_ASSERT(mcr.state == Manual_compaction_request::RUNNING); + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s started.", mcr.mc_id, + mcr.cf->GetName().c_str()); + rocksdb_manual_compactions_running++; + if (rocksdb_debug_manual_compaction_delay > 0) { + my_sleep(rocksdb_debug_manual_compaction_delay * 1000000); + } + // CompactRange may take a very long time. On clean shutdown, + // it is cancelled by CancelAllBackgroundWork, then status is + // set to shutdownInProgress. + const rocksdb::Status s = rdb->CompactRange( + getCompactRangeOptions(mcr.concurrency), mcr.cf, mcr.start, mcr.limit); + rocksdb_manual_compactions_running--; + if (s.ok()) { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s ended.", mcr.mc_id, + mcr.cf->GetName().c_str()); + } else { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s aborted. %s", + mcr.mc_id, mcr.cf->GetName().c_str(), s.getState()); + if (!s.IsShutdownInProgress()) { + rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + } else { + DBUG_ASSERT(m_requests.size() == 1); + } + } + rocksdb_manual_compactions_processed++; + clear_manual_compaction_request(mcr.mc_id, false); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + } + clear_all_manual_compaction_requests(); + DBUG_ASSERT(m_requests.empty()); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + mysql_mutex_destroy(&m_mc_mutex); + } + + void Rdb_manual_compaction_thread::clear_all_manual_compaction_requests() { + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + m_requests.clear(); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + } + + void Rdb_manual_compaction_thread::clear_manual_compaction_request( + int mc_id, bool init_only) { + bool erase = true; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + if (init_only) { + Manual_compaction_request mcr = it->second; + if (mcr.state != Manual_compaction_request::INITED) { + erase = false; + } + } + if (erase) { + m_requests.erase(it); + } + } else { + // Current code path guarantees that erasing by the same mc_id happens + // at most once. INITED state may be erased by a thread that requested + // the compaction. RUNNING state is erased by mc thread only. + DBUG_ASSERT(0); + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + } + + int Rdb_manual_compaction_thread::request_manual_compaction( + rocksdb::ColumnFamilyHandle *cf, rocksdb::Slice *start, + rocksdb::Slice *limit, int concurrency) { + int mc_id = -1; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.size() >= rocksdb_max_manual_compactions) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + Manual_compaction_request mcr; + mc_id = mcr.mc_id = ++m_latest_mc_id; + mcr.state = Manual_compaction_request::INITED; + mcr.cf = cf; + mcr.start = start; + mcr.limit = limit; + mcr.concurrency = concurrency; + m_requests.insert(std::make_pair(mcr.mc_id, mcr)); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + + bool Rdb_manual_compaction_thread::is_manual_compaction_finished(int mc_id) { + bool finished = false; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.count(mc_id) == 0) { + finished = true; + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return finished; + } + + bool ha_rocksdb::check_bloom_and_set_bounds( + THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, + const bool use_all_keys, size_t bound_len, uchar *const lower_bound, + uchar *const upper_bound, rocksdb::Slice *lower_bound_slice, + rocksdb::Slice *upper_bound_slice) { bool can_use_bloom = can_use_bloom_filter(thd, kd, eq_cond, use_all_keys); if (!can_use_bloom) { - setup_iterator_bounds(kd, eq_cond, - lower_bound_buf, upper_bound_buf, + setup_iterator_bounds(kd, eq_cond, bound_len, lower_bound, upper_bound, + lower_bound_slice, upper_bound_slice); + out_lower_bound, out_upper_bound); } return can_use_bloom; } @@@ -12973,11 -13000,6 +13561,8 @@@ const char *get_rdb_io_error_string(con // In case of core dump generation we want this function NOT to be optimized // so that we can capture as much data as possible to debug the root cause // more efficiently. +#ifdef __GNUC__ - #pragma GCC push_options - #pragma GCC optimize("O0") +#endif - void rdb_handle_io_error(const rocksdb::Status status, const RDB_IO_ERROR_TYPE err_type) { if (status.IsIOError()) { @@@ -13026,10 -13051,7 +13614,8 @@@ } } } - +#ifdef __GNUC__ - #pragma GCC pop_options +#endif - Rdb_dict_manager *rdb_get_dict_manager(void) { return &dict_manager; } Rdb_ddl_manager *rdb_get_ddl_manager(void) { return &ddl_manager; } @@@ -13573,7 -13605,6 +14202,7 @@@ maria_declare_plugin(rocksdb_se) myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, myrocks::rdb_i_s_cfoptions, myrocks::rdb_i_s_compact_stats, myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, - myrocks::rdb_i_s_index_file_map, myrocks::rdb_i_s_lock_info, - myrocks::rdb_i_s_trx_info, + myrocks::rdb_i_s_sst_props, myrocks::rdb_i_s_index_file_map, + myrocks::rdb_i_s_lock_info, myrocks::rdb_i_s_trx_info, - myrocks::rdb_i_s_deadlock_info mysql_declare_plugin_end; + myrocks::rdb_i_s_deadlock_info +maria_declare_plugin_end; diff --cc storage/rocksdb/ha_rocksdb.h index 739a2c8,ca14165..c444710 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@@ -1402,8 -1369,7 +1407,8 @@@ public virtual void rpl_after_delete_rows() override; virtual void rpl_before_update_rows() override; virtual void rpl_after_update_rows() override; - virtual bool use_read_free_rpl(); + virtual bool use_read_free_rpl() override; +#endif // MARIAROCKS_NOT_YET private: /* Flags tracking if we are inside different replication operation */ diff --cc storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result index 08f2329,9731007..0617232 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result @@@ -6,18 -6,16 +6,18 @@@ LOAD DATA INFILE <input_file> INTO TABL set rocksdb_bulk_load=0; select count(pk) from t1; count(pk) -3000000 +300000 select count(a) from t1; count(a) -3000000 +300000 select count(b) from t1; count(b) -3000000 +300000 ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; - ERROR HY000: Status error 10 received from RocksDB: Operation aborted: Failed to acquire lock due to max_num_locks limit + ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to max_num_locks limit' from ROCKSDB set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=0; diff --cc storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result index a8d5c07,6f811e1..de8a1fb --- a/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result @@@ -276,16 -276,12 +276,16 @@@ 10 20 30 - connect con1,localhost,root,,; - connection con1; alter table t1 force; - connection default; + select * from t1; + a + insert into t1 values (100); select * from t1; a +connection con1; +insert into t1 values (100); +select * from t1; +a 1 2 100 diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result index fe08cd7,8b51c12..604e557 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result @@@ -56,52 -56,44 +56,44 @@@ insert into t values () begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_before"; +set debug_dbug="+d,crash_commit_before"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After engine prepare begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_prepare"; +set debug_dbug="+d,crash_commit_after_prepare"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 4 - select max(i) from t; - max(i) - 3 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After binlog begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after_log"; +set debug_dbug="+d,crash_commit_after_log"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 6 - select max(i) from t; - max(i) - 5 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 # After everything begin; insert into t values (); insert into t values (); -set debug="+d,crash_commit_after"; +set debug_dbug="+d,crash_commit_after"; commit; ERROR HY000: Lost connection to MySQL server during query - select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; - table_schema table_name auto_increment - test t 8 - select max(i) from t; - max(i) - 7 + select max(i) into @row_max from t; + select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; + table_schema table_name auto_increment > @row_max + test t 1 drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 0c49622,17e6bed..f59b841 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@@ -150,11 -151,11 +150,29 @@@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +#--------------------------------------------------------------- +# MDEV-16703 Assertion failed in load_auto_incr_value_from_index +#--------------------------------------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++#---------------------------------- ++# Issue #792 Crash in autoincrement ++#---------------------------------- ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; + #---------------------------------- + # Issue #869 Crash in autoincrement + #---------------------------------- + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/collation.result index e372cbe,b4354ad..10e0d9b --- a/storage/rocksdb/mysql-test/rocksdb/r/collation.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/collation.result @@@ -1,11 -1,12 +1,12 @@@ - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - DROP TABLE IF EXISTS t1; + call mtr.add_suppression("Invalid pattern"); CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). DROP TABLE t1; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (latin1_bin, binary, utf8_bin). CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.t1.value3 Use binary collation (latin1_bin, binary, utf8_bin). SET GLOBAL rocksdb_strict_collation_check=0; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value3(50))) engine=rocksdb charset utf8; DROP TABLE t1; @@@ -124,7 -125,19 +126,19 @@@ ERROR HY000: Unsupported collation on s SET GLOBAL rocksdb_strict_collation_exceptions="abc"; CREATE TABLE abc (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; CREATE TABLE abcd (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; -ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (binary, latin1_bin, utf8_bin). +ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (latin1_bin, binary, utf8_bin). DROP TABLE abc; SET GLOBAL rocksdb_strict_collation_exceptions=null; - SET GLOBAL rocksdb_strict_collation_exceptions=@start_global_value; + SET GLOBAL rocksdb_strict_collation_check=1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; + ALTER TABLE t1 ADD INDEX (value); + Warnings: + Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). + DROP TABLE t1; + CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset latin1 collate latin1_bin; + ALTER TABLE t1 collate=latin1_general_ci; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result index 36db920,2a00fcb..0619e2d --- a/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/i_s_deadlock.result @@@ -13,8 -9,9 +13,9 @@@ connection default show create table information_schema.rocksdb_deadlock; Table Create Table ROCKSDB_DEADLOCK CREATE TEMPORARY TABLE `ROCKSDB_DEADLOCK` ( - `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT '0', + `DEADLOCK_ID` bigint(8) NOT NULL DEFAULT 0, + `TIMESTAMP` bigint(8) NOT NULL DEFAULT '0', - `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT '0', + `TRANSACTION_ID` bigint(8) NOT NULL DEFAULT 0, `CF_NAME` varchar(193) NOT NULL DEFAULT '', `WAITING_KEY` varchar(513) NOT NULL DEFAULT '', `LOCK_TYPE` varchar(193) NOT NULL DEFAULT '', @@@ -25,9 -22,8 +26,9 @@@ create table t (i int primary key) engine=rocksdb; insert into t values (1), (2), (3); select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #1 +connection con1; begin; select * from t where i=1 for update; i @@@ -47,13 -39,11 +48,13 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 Deadlock #2 +connection con1; begin; select * from t where i=1 for update; i @@@ -73,16 -59,14 +74,16 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 10; Deadlock #3 +connection con1; begin; select * from t where i=1 for update; i @@@ -102,24 -82,21 +103,24 @@@ connection con1 i 2 rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 set global rocksdb_max_latest_deadlocks = 1; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 1 +connection con3; set rocksdb_deadlock_detect_depth = 2; Deadlock #4 +connection con1; begin; select * from t where i=1 for update; i @@@ -150,12 -120,10 +151,12 @@@ connection con1 i 2 rollback; +connection default; set global rocksdb_max_latest_deadlocks = 5; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK Deadlock #5 +connection con1; begin; select * from t where i=1 for update; i @@@ -191,16 -154,11 +192,16 @@@ connection con1 i 2 rollback; +connection con3; rollback; +connection default; select * from information_schema.rocksdb_deadlock; - DEADLOCK_ID TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 - DEADLOCK_ID TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID CF_NAME WAITING_KEY LOCK_TYPE INDEX_NAME TABLE_NAME ROLLED_BACK + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY EXCLUSIVE PRIMARY test.t 0 + DEADLOCK_ID TIMESTAMP TRANSACTION_ID default WAITING_KEY SHARED PRIMARY test.t 1 +disconnect con1; +disconnect con2; +disconnect con3; set global rocksdb_lock_wait_timeout = @prior_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_deadlock_detect; drop table t; diff --cc storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result index 4370715,80d95e4..e024873 --- a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result @@@ -266,8 -266,8 +266,8 @@@ explai select * from t0,t1 where (t0.key1=t1.key1) and (t0.key1=3 or t0.key2<4) and t1.key1=2; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t0 ref i1,i2 i1 4 const 1 Using where + 1 SIMPLE t0 ref i1,i2 i1 4 const 2 Using where -1 SIMPLE t1 ref i1 i1 4 const 2 NULL +1 SIMPLE t1 ref i1 i1 4 const 1 explain select * from t0,t1 where t0.key1 = 5 and (t1.key1 = t0.key1 or t1.key8 = t0.key1); id select_type table type possible_keys key key_len ref rows Extra @@@ -281,12 -281,13 +281,12 @@@ id select_type table type possible_key explain select * from t1 where key1=3 or key2=4 union select * from t1 where key1<4 or key3=5; id select_type table type possible_keys key key_len ref rows Extra - 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where - 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 2 Using sort_union(i1,i3); Using where + 1 PRIMARY t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where + 2 UNION t1 index_merge i1,i3 i1,i3 4,4 NULL 4 Using sort_union(i1,i3); Using where -NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL Using temporary +NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL explain select * from (select * from t1 where key1 = 3 or key2 =3) as Z where key8 >5; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY <derived2> ALL NULL NULL NULL NULL 4 Using where -2 DERIVED t1 index_merge i1,i2 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where +1 SIMPLE t1 index_merge i1,i2,i8 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where create table t3 like t0; insert into t3 select * from t0; alter table t3 add key9 int not null, add index i9(key9); @@@ -1321,16 -1324,16 +1321,16 @@@ id select_type table type possible_key 1 SIMPLE t1 ref key1 key1 4 const ROWS Using where explain select * from t1 where pk1 < 7500 and key1 = 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 PRIMARY 4 NULL ROWS Using where +1 SIMPLE t1 range PRIMARY,key1 key1 8 NULL ROWS Using index condition explain select * from t1 where pktail1ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail1ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail1ok key1,pktail1ok 4,4 NULL 1 Using intersect(key1,pktail1ok); Using where explain select * from t1 where pktail2ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 ref key1,pktail2ok key1 4 const 1 Using where + 1 SIMPLE t1 index_merge key1,pktail2ok key1,pktail2ok 4,4 NULL 1 Using intersect(key1,pktail2ok); Using where explain select * from t1 where (pktail2ok=1 and pk1< 50000) or key1=10; id select_type table type possible_keys key key_len ref rows Extra - 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok pktail2ok,key1 8,4 NULL ROWS Using sort_union(pktail2ok,key1); Using where + 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok PRIMARY,key1 4,4 NULL ROWS Using union(PRIMARY,key1); Using where explain select * from t1 where pktail3bad=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref key1,pktail3bad EITHER_KEY 4 const ROWS Using where diff --cc storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index aba14e3,f830440..e22a85f --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@@ -16,8 -12,12 +16,9 @@@ SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WOR select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index dc13bb3,f9a3f81..c6e15dc --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@@ -915,7 -904,8 +918,8 @@@ rocksdb_enable_ttl O rocksdb_enable_ttl_read_filtering ON rocksdb_enable_write_thread_adaptive_yield OFF rocksdb_error_if_exists OFF + rocksdb_error_on_suboptimal_collation ON -rocksdb_flush_log_at_trx_commit 1 +rocksdb_flush_log_at_trx_commit 0 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF @@@ -1457,111 -1446,117 +1465,115 @@@ drop table t0, t1 # show status like 'rocksdb%'; Variable_name Value -rocksdb_rows_deleted # -rocksdb_rows_inserted # -rocksdb_rows_read # -rocksdb_rows_updated # -rocksdb_rows_deleted_blind # -rocksdb_rows_expired # -rocksdb_rows_filtered # -rocksdb_system_rows_deleted # -rocksdb_system_rows_inserted # -rocksdb_system_rows_read # -rocksdb_system_rows_updated # -rocksdb_memtable_total # -rocksdb_memtable_unflushed # -rocksdb_queries_point # -rocksdb_queries_range # -rocksdb_covered_secondary_key_lookups # -rocksdb_block_cache_add # -rocksdb_block_cache_add_failures # -rocksdb_block_cache_bytes_read # -rocksdb_block_cache_bytes_write # -rocksdb_block_cache_data_add # -rocksdb_block_cache_data_bytes_insert # -rocksdb_block_cache_data_hit # -rocksdb_block_cache_data_miss # -rocksdb_block_cache_filter_add # -rocksdb_block_cache_filter_bytes_evict # -rocksdb_block_cache_filter_bytes_insert # -rocksdb_block_cache_filter_hit # -rocksdb_block_cache_filter_miss # -rocksdb_block_cache_hit # -rocksdb_block_cache_index_add # -rocksdb_block_cache_index_bytes_evict # -rocksdb_block_cache_index_bytes_insert # -rocksdb_block_cache_index_hit # -rocksdb_block_cache_index_miss # -rocksdb_block_cache_miss # -rocksdb_block_cachecompressed_hit # -rocksdb_block_cachecompressed_miss # +Rocksdb_rows_deleted # +Rocksdb_rows_inserted # +Rocksdb_rows_read # +Rocksdb_rows_updated # +Rocksdb_rows_deleted_blind # +Rocksdb_rows_expired # +Rocksdb_rows_filtered # +Rocksdb_system_rows_deleted # +Rocksdb_system_rows_inserted # +Rocksdb_system_rows_read # +Rocksdb_system_rows_updated # +Rocksdb_memtable_total # +Rocksdb_memtable_unflushed # +Rocksdb_queries_point # +Rocksdb_queries_range # +Rocksdb_covered_secondary_key_lookups # +Rocksdb_block_cache_add # +Rocksdb_block_cache_add_failures # +Rocksdb_block_cache_bytes_read # +Rocksdb_block_cache_bytes_write # +Rocksdb_block_cache_data_add # +Rocksdb_block_cache_data_bytes_insert # +Rocksdb_block_cache_data_hit # +Rocksdb_block_cache_data_miss # +Rocksdb_block_cache_filter_add # +Rocksdb_block_cache_filter_bytes_evict # +Rocksdb_block_cache_filter_bytes_insert # +Rocksdb_block_cache_filter_hit # +Rocksdb_block_cache_filter_miss # +Rocksdb_block_cache_hit # +Rocksdb_block_cache_index_add # +Rocksdb_block_cache_index_bytes_evict # +Rocksdb_block_cache_index_bytes_insert # +Rocksdb_block_cache_index_hit # +Rocksdb_block_cache_index_miss # +Rocksdb_block_cache_miss # +Rocksdb_block_cachecompressed_hit # +Rocksdb_block_cachecompressed_miss # + rocksdb_bloom_filter_full_positive # + rocksdb_bloom_filter_full_true_positive # -rocksdb_bloom_filter_prefix_checked # -rocksdb_bloom_filter_prefix_useful # -rocksdb_bloom_filter_useful # -rocksdb_bytes_read # -rocksdb_bytes_written # -rocksdb_compact_read_bytes # -rocksdb_compact_write_bytes # -rocksdb_compaction_key_drop_new # -rocksdb_compaction_key_drop_obsolete # -rocksdb_compaction_key_drop_user # -rocksdb_flush_write_bytes # -rocksdb_get_hit_l0 # -rocksdb_get_hit_l1 # -rocksdb_get_hit_l2_and_up # -rocksdb_getupdatessince_calls # -rocksdb_git_date # -rocksdb_git_hash # -rocksdb_iter_bytes_read # +Rocksdb_bloom_filter_prefix_checked # +Rocksdb_bloom_filter_prefix_useful # +Rocksdb_bloom_filter_useful # +Rocksdb_bytes_read # +Rocksdb_bytes_written # +Rocksdb_compact_read_bytes # +Rocksdb_compact_write_bytes # +Rocksdb_compaction_key_drop_new # +Rocksdb_compaction_key_drop_obsolete # +Rocksdb_compaction_key_drop_user # +Rocksdb_flush_write_bytes # +Rocksdb_get_hit_l0 # +Rocksdb_get_hit_l1 # +Rocksdb_get_hit_l2_and_up # +Rocksdb_getupdatessince_calls # +Rocksdb_iter_bytes_read # +Rocksdb_memtable_hit # +Rocksdb_memtable_miss # +Rocksdb_no_file_closes # +Rocksdb_no_file_errors # + rocksdb_manual_compactions_processed # + rocksdb_manual_compactions_running # -rocksdb_memtable_hit # -rocksdb_memtable_miss # -rocksdb_no_file_closes # -rocksdb_no_file_errors # -rocksdb_no_file_opens # -rocksdb_num_iterators # -rocksdb_number_block_not_compressed # -rocksdb_number_db_next # -rocksdb_number_db_next_found # -rocksdb_number_db_prev # -rocksdb_number_db_prev_found # -rocksdb_number_db_seek # -rocksdb_number_db_seek_found # -rocksdb_number_deletes_filtered # -rocksdb_number_keys_read # -rocksdb_number_keys_updated # -rocksdb_number_keys_written # -rocksdb_number_merge_failures # -rocksdb_number_multiget_bytes_read # -rocksdb_number_multiget_get # -rocksdb_number_multiget_keys_read # -rocksdb_number_reseeks_iteration # -rocksdb_number_sst_entry_delete # -rocksdb_number_sst_entry_merge # -rocksdb_number_sst_entry_other # -rocksdb_number_sst_entry_put # -rocksdb_number_sst_entry_singledelete # -rocksdb_number_superversion_acquires # -rocksdb_number_superversion_cleanups # -rocksdb_number_superversion_releases # -rocksdb_row_lock_deadlocks # -rocksdb_row_lock_wait_timeouts # -rocksdb_snapshot_conflict_errors # -rocksdb_stall_l0_file_count_limit_slowdowns # -rocksdb_stall_locked_l0_file_count_limit_slowdowns # -rocksdb_stall_l0_file_count_limit_stops # -rocksdb_stall_locked_l0_file_count_limit_stops # -rocksdb_stall_pending_compaction_limit_stops # -rocksdb_stall_pending_compaction_limit_slowdowns # -rocksdb_stall_memtable_limit_stops # -rocksdb_stall_memtable_limit_slowdowns # -rocksdb_stall_total_stops # -rocksdb_stall_total_slowdowns # -rocksdb_stall_micros # -rocksdb_wal_bytes # -rocksdb_wal_group_syncs # -rocksdb_wal_synced # -rocksdb_write_other # -rocksdb_write_self # -rocksdb_write_timedout # -rocksdb_write_wal # +Rocksdb_no_file_opens # +Rocksdb_num_iterators # +Rocksdb_number_block_not_compressed # +Rocksdb_number_db_next # +Rocksdb_number_db_next_found # +Rocksdb_number_db_prev # +Rocksdb_number_db_prev_found # +Rocksdb_number_db_seek # +Rocksdb_number_db_seek_found # +Rocksdb_number_deletes_filtered # +Rocksdb_number_keys_read # +Rocksdb_number_keys_updated # +Rocksdb_number_keys_written # +Rocksdb_number_merge_failures # +Rocksdb_number_multiget_bytes_read # +Rocksdb_number_multiget_get # +Rocksdb_number_multiget_keys_read # +Rocksdb_number_reseeks_iteration # +Rocksdb_number_sst_entry_delete # +Rocksdb_number_sst_entry_merge # +Rocksdb_number_sst_entry_other # +Rocksdb_number_sst_entry_put # +Rocksdb_number_sst_entry_singledelete # +Rocksdb_number_superversion_acquires # +Rocksdb_number_superversion_cleanups # +Rocksdb_number_superversion_releases # +Rocksdb_row_lock_deadlocks # +Rocksdb_row_lock_wait_timeouts # +Rocksdb_snapshot_conflict_errors # +Rocksdb_stall_l0_file_count_limit_slowdowns # +Rocksdb_stall_locked_l0_file_count_limit_slowdowns # +Rocksdb_stall_l0_file_count_limit_stops # +Rocksdb_stall_locked_l0_file_count_limit_stops # +Rocksdb_stall_pending_compaction_limit_stops # +Rocksdb_stall_pending_compaction_limit_slowdowns # +Rocksdb_stall_memtable_limit_stops # +Rocksdb_stall_memtable_limit_slowdowns # +Rocksdb_stall_total_stops # +Rocksdb_stall_total_slowdowns # +Rocksdb_stall_micros # +Rocksdb_wal_bytes # +Rocksdb_wal_group_syncs # +Rocksdb_wal_synced # +Rocksdb_write_other # +Rocksdb_write_self # +Rocksdb_write_timedout # +Rocksdb_write_wal # select VARIABLE_NAME from INFORMATION_SCHEMA.global_status where VARIABLE_NAME LIKE 'rocksdb%'; VARIABLE_NAME ROCKSDB_ROWS_DELETED @@@ -1617,7 -1614,11 +1631,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES @@@ -1726,7 -1729,11 +1744,9 @@@ ROCKSDB_GET_HIT_L ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS -ROCKSDB_GIT_DATE -ROCKSDB_GIT_HASH ROCKSDB_ITER_BYTES_READ + ROCKSDB_MANUAL_COMPACTIONS_PROCESSED + ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS ROCKSDB_NO_FILE_CLOSES diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result index e52f495,eaa7ae4..ca1ad57 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result @@@ -50,21 -40,27 +50,33 @@@ 3 insert into t values (4), (1); ERROR 40001: Deadlock found when trying to get lock; try restarting transaction - # Statement should be rolled back + # Transaction should be rolled back select * from t; i - 3 rollback; +connection con2; i rollback; +connection con1; i rollback; +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + create table t1 (id int primary key, value int, value2 int, index(value)) engine=rocksdb; + insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); + begin; + update t1 force index (value) set value2=value2+1 where value=3; + begin; + update t1 force index (value) set value2=value2+1 where value=2; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=4; + update t1 force index (value) set value2=value2+1 where value=3; + ERROR 40001: Deadlock found when trying to get lock; try restarting transaction + rollback; + rollback; + drop table t1; set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; drop table t,r1,r2; diff --cc storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result index 24466d9,0b1d346..b096e42 --- a/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result @@@ -1,11 -1,11 +1,11 @@@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] DROP TABLE IF EXISTS t1; +connection slave; include/stop_slave.inc + create table t1 (a int, b int, primary key (a), unique key (b)) engine=rocksdb; +connection master; - create table t1 (a int) engine=rocksdb; +connection slave; show variables like 'rpl_skip_tx_api'; Variable_name Value rpl_skip_tx_api ON diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test index d715eb7,522ff9f..4f34cbe --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test @@@ -1,6 -1,6 +1,7 @@@ --source include/have_rocksdb.inc --source include/have_debug.inc +--source include/have_partition.inc + --source include/not_valgrind.inc --disable_warnings drop table if exists t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test index 5eac859,be8b9fb..61a10a6 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test @@@ -66,9 -63,7 +66,9 @@@ ALTER TABLE t1 ADD INDEX kb(b), ALGORIT --disable_warnings # now do same index using copy algorithm # hitting max row locks (1M) +set @tmp= @@rocksdb_max_row_locks; +set session rocksdb_max_row_locks=1000; - --error ER_RDB_STATUS_GENERAL + --error ER_GET_ERRMSG ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; diff --cc storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test index 86a0a7f,238e62a..f4da0b7 --- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test @@@ -117,12 -115,12 +117,35 @@@ INSERT INTO t0 VALUES(0) ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +--echo #--------------------------------------------------------------- +--echo # MDEV-16703 Assertion failed in load_auto_incr_value_from_index +--echo #--------------------------------------------------------------- + +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; ++ ++--echo #---------------------------------- ++--echo # Issue #792 Crash in autoincrement ++--echo #---------------------------------- ++ ++CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; ++INSERT INTO t1 VALUES(2177,0); ++DROP TABLE t1; ++ ++CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; ++INSERT INTO t0 VALUES(0); ++ALTER TABLE t0 AUTO_INCREMENT=0; ++DROP TABLE t0; ++ + --echo #---------------------------------- + --echo # Issue #869 Crash in autoincrement + --echo #---------------------------------- + + CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; + INSERT INTO t1 (a) VALUES (1); + UPDATE t1 SET pk = 3; + ALTER TABLE t1 AUTO_INCREMENT 2; + DROP TABLE t1; diff --cc storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test index 9e7c816,c1cdf53..53bfba3 --- a/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test @@@ -8,7 -4,8 +8,10 @@@ let $restart_file= $MYSQLTEST_VARDIR/tm let $error_log= $MYSQLTEST_VARDIR/log/my_restart.err; select variable_name, variable_value from information_schema.global_variables where variable_name="rocksdb_ignore_unknown_options"; ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ---exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" +--exec find $MYSQLD_DATADIR/#rocksdb/OPTIONS* | sort -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "sed -i 's/rocksdb_version=.*/rocksdb_version=99.9.9/' {}" ++# --exec find $MYSQLD_DATADIR/.rocksdb/OPTIONS* | sort -t- -k 2 -n | tail -1 | xargs -0 -I {} -t sh -c "echo hello=world>>{}" ++ --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --shutdown_server 10 diff --cc storage/rocksdb/mysql-test/rocksdb/t/collation.test index 7152925,38ce1b2..3b808bc --- a/storage/rocksdb/mysql-test/rocksdb/t/collation.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/collation.test @@@ -1,13 -1,7 +1,9 @@@ --source include/have_rocksdb.inc ---source include/have_fullregex.inc +# MariaDB doesn't have server variables to check for GCC version, so the +# following check is commented out: +# --source include/have_fullregex.inc - SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; - - --disable_warnings - DROP TABLE IF EXISTS t1; - --enable_warnings + call mtr.add_suppression("Invalid pattern"); # ci non-indexed column is allowed CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; diff --cc storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 67dae5d,2ae0b40..09998b9 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@@ -28,10 -17,10 +28,11 @@@ select VALUE into @keysIn from INFORMAT CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); + set global rocksdb_force_flush_memtable_now = true; ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --cc storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test index b884738,cf8fb29..f4fdaa7 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test @@@ -1,5 -1,5 +1,6 @@@ --source include/have_rocksdb.inc +--source include/have_partition.inc + --source suite/rocksdb/include/have_write_committed.inc # # RocksDB Storage Engine tests diff --cc storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test index e9dcc60,d9ffb08..9f1e28d --- a/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test @@@ -43,41 -46,47 +45,44 @@@ shutdown_server 10 shutdown_server 10; # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message ---exec echo "Expect errors that we are missing two .frm files" ---let SEARCH_FILE=$LOG ---let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t1 is registered in RocksDB but does not have a .frm file +--echo "Expect errors that we are missing two .frm files" +--let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN=Schema mismatch --source include/search_pattern_in_file.inc + --let SEARCH_PATTERN=RocksDB: Schema mismatch - Table test.t2 is registered in RocksDB but does not have a .frm file + --source include/search_pattern_in_file.inc # Now shut down again and rename one the .frm file back and make a copy of it --exec echo "wait" >$_expect_file_name shutdown_server 10; - + --remove_file $LOG - # Rename the file ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm ---exec mv $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm ---exec cp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm +--move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t2.frm +--copy_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm # Attempt to restart the server - --exec echo "restart:--rocksdb_validate_tables=2" >$_expect_file_name + --exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" >$_expect_file_name ---sleep 5 --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect # We should now have an error message for the second case ---exec echo "Expect an error that we have an extra .frm file" +--echo "Expect an error that we have an extra .frm file" - --let SEARCH_PATTERN=Schema mismatch + --let SEARCH_FILE=$LOG + --let SEARCH_PATTERN=Schema mismatch - A .frm file exists for table test.t1_dummy, but that table is not registered in RocksDB --source include/search_pattern_in_file.inc # Shut down an clean up diff --cc storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc index 5e5961f,492b5cd..16ad535 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc @@@ -40,9 -40,9 +40,9 @@@ select * from mysql.slave_gtid_info --write_file $MYSQL_TMP_DIR/truncate_tail_wal.sh #!/bin/bash -F=`ls -t $slave_data_dir/.rocksdb/*.log | head -n 1` +F=`ls -t $slave_data_dir/\#rocksdb/*.log | head -n 1` SIZE=`stat -c %s $F` - NEW_SIZE=`expr $SIZE - 10` + NEW_SIZE=`expr $SIZE - 30` truncate -s $NEW_SIZE $F rc=$? if [[ $rc != 0 ]]; then diff --cc storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test index 39688e6,31b50aa..8d3bab4 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_block_cache_size_basic.test @@@ -1,7 -1,21 +1,21 @@@ --source include/have_rocksdb.inc + CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO valid_values VALUES(65536); + INSERT INTO valid_values VALUES(1024); + INSERT INTO valid_values VALUES(1*1024*1024); + + CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + INSERT INTO invalid_values VALUES('\'aaa\''); + INSERT INTO invalid_values VALUES('\'bbb\''); + INSERT INTO invalid_values VALUES('\'-1\''); + INSERT INTO invalid_values VALUES('\'101\''); + INSERT INTO invalid_values VALUES('\'484436\''); + --let $sys_var=ROCKSDB_BLOCK_CACHE_SIZE - --let $read_only=1 + --let $read_only=0 --let $session=0 ---source ../include/rocksdb_sys_var.inc +--source include/rocksdb_sys_var.inc + DROP TABLE valid_values; + DROP TABLE invalid_values; diff --cc storage/rocksdb/rdb_i_s.cc index de33d69,147ad7f..9192795 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@@ -1757,10 -1869,26 +1906,26 @@@ struct st_maria_plugin rdb_i_s_ddl = nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ + 0, /* flags */ + }; + + struct st_mysql_plugin rdb_i_s_sst_props = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_SST_PROPS", + "Facebook", + "RocksDB SST Properties", + PLUGIN_LICENSE_GPL, + rdb_i_s_sst_props_init, + rdb_i_s_deinit, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; -struct st_mysql_plugin rdb_i_s_index_file_map = { +struct st_maria_plugin rdb_i_s_index_file_map = { MYSQL_INFORMATION_SCHEMA_PLUGIN, &rdb_i_s_info, "ROCKSDB_INDEX_FILE_MAP", diff --cc storage/rocksdb/rdb_i_s.h index d6a48bf,dd9f4a9..f4c4b04 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@@ -21,16 -21,17 +21,17 @@@ namespace myrocks Declare INFORMATION_SCHEMA (I_S) plugins needed by MyRocks storage engine. */ -extern struct st_mysql_plugin rdb_i_s_cfstats; -extern struct st_mysql_plugin rdb_i_s_dbstats; -extern struct st_mysql_plugin rdb_i_s_perf_context; -extern struct st_mysql_plugin rdb_i_s_perf_context_global; -extern struct st_mysql_plugin rdb_i_s_cfoptions; -extern struct st_mysql_plugin rdb_i_s_compact_stats; -extern struct st_mysql_plugin rdb_i_s_global_info; -extern struct st_mysql_plugin rdb_i_s_ddl; -extern struct st_mysql_plugin rdb_i_s_sst_props; -extern struct st_mysql_plugin rdb_i_s_index_file_map; -extern struct st_mysql_plugin rdb_i_s_lock_info; -extern struct st_mysql_plugin rdb_i_s_trx_info; -extern struct st_mysql_plugin rdb_i_s_deadlock_info; +extern struct st_maria_plugin rdb_i_s_cfstats; +extern struct st_maria_plugin rdb_i_s_dbstats; +extern struct st_maria_plugin rdb_i_s_perf_context; +extern struct st_maria_plugin rdb_i_s_perf_context_global; +extern struct st_maria_plugin rdb_i_s_cfoptions; +extern struct st_maria_plugin rdb_i_s_compact_stats; +extern struct st_maria_plugin rdb_i_s_global_info; +extern struct st_maria_plugin rdb_i_s_ddl; ++extern struct st_maria_plugin rdb_i_s_sst_props; +extern struct st_maria_plugin rdb_i_s_index_file_map; +extern struct st_maria_plugin rdb_i_s_lock_info; +extern struct st_maria_plugin rdb_i_s_trx_info; +extern struct st_maria_plugin rdb_i_s_deadlock_info; } // namespace myrocks diff --cc storage/rocksdb/rocksdb index ba295cd,0000000..926f3a7 mode 160000,000000..160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@@ -1,1 -1,0 +1,1 @@@ - Subproject commit ba295cda29daee3ffe58549542804efdfd969784 ++Subproject commit 926f3a78a64b327475ee6c60b6c8ab4f34253204

1 0

[Commits] 880a22e5c4a: MDEV-15306: Wrong/Unexpected result with the value optimizer_use_condition_selectivity set to 4
by Varun 27 Aug '18

27 Aug '18

revision-id: 880a22e5c4aa7239b3244ba1b83854955295e849 (mariadb-10.0.36-11-g880a22e5c4a) parent(s): 5fb251642e19c59306c32333dbba669ce160457c author: Varun Gupta committer: Varun Gupta timestamp: 2018-08-27 20:41:35 +0530 message: MDEV-15306: Wrong/Unexpected result with the value optimizer_use_condition_selectivity set to 4 Currently for selectivity calculation we perform range analysis for a column even when we don't have any statistics(EITS). This makes less sense but is used to catch contradiction for WHERE condition. So the solution is to not perform range analysis for selectivity calculation for columns that do not have statistics. --- mysql-test/r/selectivity.result | 44 ++++++++++++++++++++++++++++++++-- mysql-test/r/selectivity_innodb.result | 44 ++++++++++++++++++++++++++++++++-- mysql-test/t/selectivity.test | 36 ++++++++++++++++++++++++++++ sql/opt_range.cc | 10 ++++++-- sql/sql_statistics.h | 29 ++++++++++++++++++---- 5 files changed, 152 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 3e8fb8e2e41..6af4f9a9ace 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -782,9 +782,9 @@ set optimizer_use_condition_selectivity=3; explain extended select * from t1 where a < 1 and a > 7; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where Warnings: -Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where 0 +Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where ((`test`.`t1`.`a` < 1) and (`test`.`t1`.`a` > 7)) select * from t1 where a < 1 and a > 7; a drop table t1; @@ -1600,3 +1600,43 @@ drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# MDEV-15306: Wrong/Unexpected result with the value +# optimizer_use_condition_selectivity set to 4 +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); +CREATE FUNCTION f1() RETURNS INT DETERMINISTIC +BEGIN +SET @cnt := @cnt + 1; +RETURN 1; +END;| +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@use_stat_tables='complementary'; +set @@optimizer_use_condition_selectivity=4; +SET @cnt= 0; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +1 +set @@use_stat_tables='preferably'; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SET @cnt := 0; +set @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +2 +alter table t1 force; +set @@use_stat_tables= @save_use_stat_tables; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +drop table t1; +drop function f1; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 748ef0cb6ca..e0ed2865f13 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -789,9 +789,9 @@ set optimizer_use_condition_selectivity=3; explain extended select * from t1 where a < 1 and a > 7; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where Warnings: -Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where 0 +Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where ((`test`.`t1`.`a` < 1) and (`test`.`t1`.`a` > 7)) select * from t1 where a < 1 and a > 7; a drop table t1; @@ -1604,6 +1604,46 @@ drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# MDEV-15306: Wrong/Unexpected result with the value +# optimizer_use_condition_selectivity set to 4 +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); +CREATE FUNCTION f1() RETURNS INT DETERMINISTIC +BEGIN +SET @cnt := @cnt + 1; +RETURN 1; +END;| +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@use_stat_tables='complementary'; +set @@optimizer_use_condition_selectivity=4; +SET @cnt= 0; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +1 +set @@use_stat_tables='preferably'; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SET @cnt := 0; +set @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +2 +alter table t1 force; +set @@use_stat_tables= @save_use_stat_tables; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +drop table t1; +drop function f1; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; set @tmp_oucs= @@optimizer_use_condition_selectivity; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index afaa937c360..557a4e92bca 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1064,3 +1064,39 @@ drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; + +--echo # +--echo # MDEV-15306: Wrong/Unexpected result with the value +--echo # optimizer_use_condition_selectivity set to 4 +--echo # + +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); + +delimiter |; +CREATE FUNCTION f1() RETURNS INT DETERMINISTIC +BEGIN + SET @cnt := @cnt + 1; + RETURN 1; +END;| +delimiter ;| +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@use_stat_tables='complementary'; +set @@optimizer_use_condition_selectivity=4; +SET @cnt= 0; +SELECT * FROM t1 WHERE a = f1(); +SELECT @cnt; + +set @@use_stat_tables='preferably'; +analyze table t1 persistent for all; +SET @cnt := 0; +set @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a = f1(); +SELECT @cnt; +alter table t1 force; +set @@use_stat_tables= @save_use_stat_tables; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +drop table t1; +drop function f1; + diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 734adfbc9d1..ceabac2b744 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -3326,13 +3326,18 @@ bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param, for (field_ptr= table->field; *field_ptr; field_ptr++) { - if (bitmap_is_set(used_fields, (*field_ptr)->field_index)) + Column_statistics* col_stats= (*field_ptr)->read_stats; + if (bitmap_is_set(used_fields, (*field_ptr)->field_index) + && col_stats && !col_stats->no_stat_values_provided()) parts++; } KEY_PART *key_part; uint keys= 0; + if (!parts) + return TRUE; + if (!(key_part= (KEY_PART *) alloc_root(param->mem_root, sizeof(KEY_PART) * parts))) return TRUE; @@ -3628,7 +3633,8 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond) */ if (thd->variables.optimizer_use_condition_selectivity > 2 && - !bitmap_is_clear_all(used_fields)) + !bitmap_is_clear_all(used_fields) && + thd->variables.use_stat_tables > 0) { PARAM param; MEM_ROOT alloc; diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 6a43e42ab96..a43e0caf0fe 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -342,12 +342,17 @@ class Column_statistics public: Histogram histogram; + + uint32 default_value() + { + return + ((1 << (COLUMN_STAT_HISTOGRAM-COLUMN_STAT_COLUMN_NAME))-1) << + (COLUMN_STAT_COLUMN_NAME+1); + } void set_all_nulls() { - column_stat_nulls= - ((1 << (COLUMN_STAT_HISTOGRAM-COLUMN_STAT_COLUMN_NAME))-1) << - (COLUMN_STAT_COLUMN_NAME+1); + column_stat_nulls= default_value(); } void set_not_null(uint stat_field_no) @@ -393,8 +398,22 @@ class Column_statistics bool min_max_values_are_provided() { return !is_null(COLUMN_STAT_MIN_VALUE) && - !is_null(COLUMN_STAT_MIN_VALUE); - } + !is_null(COLUMN_STAT_MAX_VALUE); + } + /* + This function checks whether the values for the fields of the statistical + tables that were NULL by DEFAULT for a column have changed or not. + + @retval + TRUE: Statistics are not present for a column + FALSE: Statisitics are present for a column + */ + bool no_stat_values_provided() + { + if (column_stat_nulls == default_value()) + return true; + return false; + } };

1 0

[Commits] eb29f64: MDEV-17017 Explain for query using derived table specified with
by IgorBabaev 27 Aug '18

27 Aug '18

revision-id: eb29f64b998d3ab9817b525babb175c4f600da0d (mariadb-10.3.7-145-geb29f64) parent(s): 6c6ca907ee84e3f129bc33be8202c0fee2f29196 author: Igor Babaev committer: Igor Babaev timestamp: 2018-08-27 08:12:59 -0700 message: MDEV-17017 Explain for query using derived table specified with a table value constructor shows wrong number of rows This is another attempt to fix this bug. The previous patch did not take into account that a transformation for ALL/ANY subqueries could be applied to the materialized table that wrapped the table value constructor used as a specification of the subselect used an ALL/ANY subquery. In this case the result of the derived table used a sink of the class select_subselect rather than of the class select_unit. Thus the previous fix could cause memory overwrites when running EXPLAIN for queries with table value constructors in ALL/ANY subselects. --- mysql-test/main/table_value_constr.test | 2 -- sql/sql_class.h | 5 ++--- sql/sql_lex.cc | 6 +++--- sql/table.cc | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/mysql-test/main/table_value_constr.test b/mysql-test/main/table_value_constr.test index 5df40d10..0dd0a7a 100644 --- a/mysql-test/main/table_value_constr.test +++ b/mysql-test/main/table_value_constr.test @@ -1123,5 +1123,3 @@ PREPARE stmt FROM "SELECT * FROM (VALUES(1 + 1,2,'abc')) t"; EXECUTE stmt; EXECUTE stmt; DEALLOCATE PREPARE stmt; - - diff --git a/sql/sql_class.h b/sql/sql_class.h index faae443..026ef4d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4956,7 +4956,8 @@ class select_result :public select_result_sink SELECT_LEX_UNIT *unit; /* Something used only by the parser: */ public: - select_result(THD *thd_arg): select_result_sink(thd_arg) {} + ha_rows est_records; /* estimated number of records in the result */ + select_result(THD *thd_arg): select_result_sink(thd_arg), est_records(0) {} void set_unit(SELECT_LEX_UNIT *unit_arg) { unit= unit_arg; } virtual ~select_result() {}; /** @@ -5528,7 +5529,6 @@ class select_unit :public select_result_interceptor TMP_TABLE_PARAM tmp_table_param; int write_err; /* Error code from the last send_data->ha_write_row call. */ TABLE *table; - ha_rows records; select_unit(THD *thd_arg): select_result_interceptor(thd_arg), @@ -5566,7 +5566,6 @@ class select_unit :public select_result_interceptor curr_sel= UINT_MAX; step= UNION_TYPE; write_err= 0; - records= 0; } void change_select(); }; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 7d88a6c..1b1031d 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -4678,18 +4678,18 @@ void SELECT_LEX::increase_derived_records(ha_rows records) return; } - select_unit *result= (select_unit*)unit->result; + select_result *result= unit->result; switch (linkage) { case INTERSECT_TYPE: // result of intersect can't be more then one of components - set_if_smaller(result->records, records); + set_if_smaller(result->est_records, records); case EXCEPT_TYPE: // in worse case none of record will be removed break; default: // usual UNION - result->records+= records; + result->est_records+= records; break; } } diff --git a/sql/table.cc b/sql/table.cc index 627f615..4aef9aa 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -8334,7 +8334,7 @@ int TABLE_LIST::fetch_number_of_rows() } if (is_materialized_derived() && !fill_me) { - table->file->stats.records= ((select_unit*)(get_unit()->result))->records; + table->file->stats.records= get_unit()->result->est_records; set_if_bigger(table->file->stats.records, 2); table->used_stat_records= table->file->stats.records; }

1 0

[Commits] 321e6784546: cleanup
by jan 27 Aug '18

27 Aug '18

revision-id: 321e6784546e249f9ba106cc66b5e8d85e5173a9 (mariadb-10.3.7-145-g321e6784546) parent(s): 095de15d753807ee78e97c2a1ccfc688abb05ee7 author: Jan Lindström committer: Jan Lindström timestamp: 2018-08-27 16:07:20 +0300 message: cleanup --- mysql-test/suite/galera/r/MW-336.result | 23 +++++++++++++++++++++++ mysql-test/suite/galera/t/MW-336.test | 11 +++++++++++ 2 files changed, 34 insertions(+) diff --git a/mysql-test/suite/galera/r/MW-336.result b/mysql-test/suite/galera/r/MW-336.result index fa5514dfc47..1b5d2984e39 100644 --- a/mysql-test/suite/galera/r/MW-336.result +++ b/mysql-test/suite/galera/r/MW-336.result @@ -74,4 +74,27 @@ INSERT INTO t1 VALUES (1); INSERT INTO t1 VALUES (1); connection node_1; SET GLOBAL wsrep_slave_threads = 1; +connection node_2; +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +connection node_1; DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MW-336.test b/mysql-test/suite/galera/t/MW-336.test index 7af039e577b..4bdbfb2ecff 100644 --- a/mysql-test/suite/galera/t/MW-336.test +++ b/mysql-test/suite/galera/t/MW-336.test @@ -66,4 +66,15 @@ while ($count) --source include/wait_condition.inc SET GLOBAL wsrep_slave_threads = 1; + +--connection node_2 +# Generate 21 replication events +--let $count = 21 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} + +--connection node_1 DROP TABLE t1;

1 0

[Commits] 40d84f31c81: c
by Oleksandr Byelkin 27 Aug '18

27 Aug '18

revision-id: 40d84f31c81496f59e48cb3fa318cb0eec580bfb (mariadb-10.2.16-102-g40d84f31c81) parent(s): c634fd0cbf36605b1ec49d1ec7f613d2834e6333 author: Oleksandr Byelkin committer: Oleksandr Byelkin timestamp: 2018-08-27 13:01:02 +0200 message: c --- mysql-test/r/func_default.result | 2 +- mysql-test/r/func_time.result | 33 +++++++++++++++++++++++++++++++-- mysql-test/t/func_time.test | 15 +++++++++++++-- sql/field.cc | 8 ++------ sql/item.cc | 6 ++---- 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/mysql-test/r/func_default.result b/mysql-test/r/func_default.result index 1f331af287c..3927c387680 100644 --- a/mysql-test/r/func_default.result +++ b/mysql-test/r/func_default.result @@ -8,7 +8,7 @@ explain extended select default(str), default(strnull), default(intg), default(r id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t1 system NULL NULL NULL NULL 1 100.00 Warnings: -Note 1003 select default(`str`) AS `default(str)`,default(`strnull`) AS `default(strnull)`,default(`intg`) AS `default(intg)`,default(`rel`) AS `default(rel)` from dual +Note 1003 select default(`test`.`t1`.`str`) AS `default(str)`,default(`test`.`t1`.`strnull`) AS `default(strnull)`,default(`test`.`t1`.`intg`) AS `default(intg)`,default(`test`.`t1`.`rel`) AS `default(rel)` from dual select * from t1 where str <> default(str); str strnull intg rel 0 0 diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 5b9286323a7..1407a04e0ce 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -3288,8 +3288,37 @@ NULL NULL Warnings: Warning 1292 Incorrect datetime value: '18446744073709551615' -DROP VIEW v1; -DROP TABLE t1; +CREATE TABLE t2 (pk int default 1, a1 date); +INSERT INTO t2 VALUES (4,NULL); +CREATE view v2 as SELECT default(t1.pk), default(t2.pk), t1.pk from t1,t2; +select * from v2; +default(t1.pk) default(t2.pk) pk +0 1 1 +0 1 2 +0 1 3 +0 1 4 +show create view v2; +View Create View character_set_client collation_connection +v2 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v2` AS select default(`t1`.`pk`) AS `default(t1.pk)`,default(`t2`.`pk`) AS `default(t2.pk)`,`t1`.`pk` AS `pk` from (`t1` join `t2`) latin1 latin1_swedish_ci +CREATE view v3 as SELECT default(pk) from t2; +select * from v3; +default(pk) +1 +explain extended select * from v3; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 system NULL NULL NULL NULL 1 100.00 +Warnings: +Note 1003 select default(`test`.`t2`.`pk`) AS `default(pk)` from dual +explain extended select default(pk) from t2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 system NULL NULL NULL NULL 1 100.00 +Warnings: +Note 1003 select default(`test`.`t2`.`pk`) AS `default(pk)` from dual +show create view v3; +View Create View character_set_client collation_connection +v3 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v3` AS select default(`t2`.`pk`) AS `default(pk)` from `t2` latin1 latin1_swedish_ci +DROP VIEW v1,v2,v3; +DROP TABLE t1,t2; # # End of 10.2 tests # diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 0253a2d548e..0805063c395 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1876,8 +1876,19 @@ SELECT t1.pk AS pk, t1.a1 AS a1 FROM t1; SELECT a1 BETWEEN (('2018-08-24')) AND (DEFAULT(pk)) FROM v1; SELECT a1 BETWEEN (('2018-08-24')) AND (~ DEFAULT(pk)) FROM v1; -DROP VIEW v1; -DROP TABLE t1; +CREATE TABLE t2 (pk int default 1, a1 date); +INSERT INTO t2 VALUES (4,NULL); +CREATE view v2 as SELECT default(t1.pk), default(t2.pk), t1.pk from t1,t2; +select * from v2; +show create view v2; +CREATE view v3 as SELECT default(pk) from t2; +select * from v3; +explain extended select * from v3; +explain extended select default(pk) from t2; +show create view v3; + +DROP VIEW v1,v2,v3; +DROP TABLE t1,t2; --echo # --echo # End of 10.2 tests diff --git a/sql/field.cc b/sql/field.cc index e12adbf47b3..80553cca015 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -76,18 +76,14 @@ const char field_separator=','; (!table->read_set || \ bitmap_is_set(table->read_set, field_index) || \ (!(ptr >= table->record[0] && \ - ptr < table->record[0] + table->s->reclength) && \ - !(ptr >= table->record[1] && \ - ptr < table->record[1] + table->s->reclength)))) + ptr < table->record[0] + table->s->reclength)))) #define ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED \ DBUG_ASSERT(is_stat_field || !table || \ (!table->write_set || \ bitmap_is_set(table->write_set, field_index) || \ (!(ptr >= table->record[0] && \ - ptr < table->record[0] + table->s->reclength) && \ - !(ptr >= table->record[1] && \ - ptr < table->record[1] + table->s->reclength))) || \ + ptr < table->record[0] + table->s->reclength))) || \ (table->vcol_set && bitmap_is_set(table->vcol_set, field_index))) #define FLAGSTR(S,F) ((S) & (F) ? #F " " : "") diff --git a/sql/item.cc b/sql/item.cc index 70fea097d36..ab26f619768 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -8961,7 +8961,7 @@ bool Item_default_value::fix_fields(THD *thd, Item **items) */ fix_session_vcol_expr_for_read(thd, def_field, def_field->default_value); if (thd->mark_used_columns != MARK_COLUMNS_NONE) - def_field->default_value->expr->walk(&Item::register_field_in_read_map, 1, 0); + def_field->default_value->expr->update_used_tables(); def_field->move_field(newptr+1, def_field->maybe_null() ? newptr : 0, 1); } else @@ -8990,9 +8990,7 @@ void Item_default_value::print(String *str, enum_query_type query_type) tables but its name so we set QT_NO_DATA_EXPANSION (as we print for table definition, also we do not need table and database name) */ - query_type= (enum_query_type) (query_type | QT_NO_DATA_EXPANSION | - QT_ITEM_IDENT_SKIP_DB_NAMES | - QT_ITEM_IDENT_SKIP_TABLE_NAMES); + query_type= (enum_query_type) (query_type | QT_NO_DATA_EXPANSION); arg->print(str, query_type); str->append(')'); }

1 0

[Commits] e13108c7fcf: MDEV-15306: Wrong/Unexpected result with the value optimizer_use_condition_selectivity set to 4
by varunraiko1803＠gmail.com 27 Aug '18

27 Aug '18

revision-id: e13108c7fcf472faa7eaba1a7083d80c9bad4f55 (mariadb-10.3.0-696-ge13108c7fcf) parent(s): 87ee85634cae8538b922b6eb3d275a5f4343dbb1 author: Varun Gupta committer: Varun Gupta timestamp: 2018-03-30 04:50:47 +0530 message: MDEV-15306: Wrong/Unexpected result with the value optimizer_use_condition_selectivity set to 4 Currently we perform range analysis for a column even when we don't have any statistics(EITS). This makes less sense but is used to catch contradiction for WHERE condition. So the solution is to not perform range analysis for columns that do not have statistics that can be used by the optimizer. --- mysql-test/main/selectivity.result | 48 ++++++++++++++++++++++++++++--- mysql-test/main/selectivity.test | 37 ++++++++++++++++++++++++ mysql-test/main/selectivity_innodb.result | 48 ++++++++++++++++++++++++++++--- sql/opt_range.cc | 10 +++++-- sql/sql_statistics.h | 17 ++++++++++- 5 files changed, 149 insertions(+), 11 deletions(-) diff --git a/mysql-test/main/selectivity.result b/mysql-test/main/selectivity.result index 9cb6ee3e9bf..cda2c28fa16 100644 --- a/mysql-test/main/selectivity.result +++ b/mysql-test/main/selectivity.result @@ -782,9 +782,9 @@ set optimizer_use_condition_selectivity=3; explain extended select * from t1 where a < 1 and a > 7; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where Warnings: -Note 1003 select 7 AS `a` from `test`.`t1` where 0 +Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` < 1 and `test`.`t1`.`a` > 7 select * from t1 where a < 1 and a > 7; a drop table t1; @@ -1506,9 +1506,9 @@ col1 explain extended select * from t2 where col1 < 'b' and col1 > 'd'; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +1 SIMPLE t2 ALL NULL NULL NULL NULL 8 100.00 Using where Warnings: -Note 1003 select 'd' AS `col1` from `test`.`t2` where 0 +Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where `test`.`t2`.`col1` < 'b' and `test`.`t2`.`col1` > 'd' drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; @@ -1533,3 +1533,43 @@ DROP TABLE t1; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# MDEV-15306: Wrong/Unexpected result with the value +# optimizer_use_condition_selectivity set to 4 +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); +CREATE FUNCTION f1() RETURNS INT DETERMINISTIC +BEGIN +SET @cnt := @cnt + 1; +RETURN 1; +END;| +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@use_stat_tables='complementary'; +set @@optimizer_use_condition_selectivity=4; +SET @cnt= 0; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +1 +set @@use_stat_tables='preferably'; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SET @cnt := 0; +set @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +2 +alter table t1 force; +set @@use_stat_tables= @save_use_stat_tables; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +drop table t1; +drop function f1; diff --git a/mysql-test/main/selectivity.test b/mysql-test/main/selectivity.test index 3e60f242083..77343beadce 100644 --- a/mysql-test/main/selectivity.test +++ b/mysql-test/main/selectivity.test @@ -1048,3 +1048,40 @@ DROP TABLE t1; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; + +--echo # +--echo # MDEV-15306: Wrong/Unexpected result with the value +--echo # optimizer_use_condition_selectivity set to 4 +--echo # + +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); + +delimiter |; +CREATE FUNCTION f1() RETURNS INT DETERMINISTIC +BEGIN +SET @cnt := @cnt + 1; +RETURN 1; +END;| +delimiter ;| + +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@use_stat_tables='complementary'; +set @@optimizer_use_condition_selectivity=4; +SET @cnt= 0; +SELECT * FROM t1 WHERE a = f1(); +SELECT @cnt; + +set @@use_stat_tables='preferably'; +analyze table t1 persistent for all; +SET @cnt := 0; +set @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a = f1(); +SELECT @cnt; +alter table t1 force; +set @@use_stat_tables= @save_use_stat_tables; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; + +drop table t1; +drop function f1; diff --git a/mysql-test/main/selectivity_innodb.result b/mysql-test/main/selectivity_innodb.result index 236647c6091..01783b978ff 100644 --- a/mysql-test/main/selectivity_innodb.result +++ b/mysql-test/main/selectivity_innodb.result @@ -789,9 +789,9 @@ set optimizer_use_condition_selectivity=3; explain extended select * from t1 where a < 1 and a > 7; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where Warnings: -Note 1003 select 7 AS `a` from `test`.`t1` where 0 +Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` < 1 and `test`.`t1`.`a` > 7 select * from t1 where a < 1 and a > 7; a drop table t1; @@ -1516,9 +1516,9 @@ col1 explain extended select * from t2 where col1 < 'b' and col1 > 'd'; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +1 SIMPLE t2 ALL NULL NULL NULL NULL 8 100.00 Using where Warnings: -Note 1003 select 'd' AS `col1` from `test`.`t2` where 0 +Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where `test`.`t2`.`col1` < 'b' and `test`.`t2`.`col1` > 'd' drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; @@ -1543,6 +1543,46 @@ DROP TABLE t1; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# MDEV-15306: Wrong/Unexpected result with the value +# optimizer_use_condition_selectivity set to 4 +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); +CREATE FUNCTION f1() RETURNS INT DETERMINISTIC +BEGIN +SET @cnt := @cnt + 1; +RETURN 1; +END;| +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@use_stat_tables='complementary'; +set @@optimizer_use_condition_selectivity=4; +SET @cnt= 0; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +1 +set @@use_stat_tables='preferably'; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SET @cnt := 0; +set @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a = f1(); +a +1 +SELECT @cnt; +@cnt +2 +alter table t1 force; +set @@use_stat_tables= @save_use_stat_tables; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +drop table t1; +drop function f1; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; set @tmp_oucs= @@optimizer_use_condition_selectivity; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 38dbed92a22..269d4af667d 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -2733,13 +2733,18 @@ bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param, for (field_ptr= table->field; *field_ptr; field_ptr++) { - if (bitmap_is_set(used_fields, (*field_ptr)->field_index)) + Column_statistics* col_stats= (*field_ptr)->read_stats; + if (col_stats && !col_stats->check_all_values_are_default() + && bitmap_is_set(used_fields, (*field_ptr)->field_index)) parts++; } KEY_PART *key_part; uint keys= 0; + if(!parts) + return TRUE; + if (!(key_part= (KEY_PART *) alloc_root(param->mem_root, sizeof(KEY_PART) * parts))) return TRUE; @@ -3025,7 +3030,8 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond) */ if (thd->variables.optimizer_use_condition_selectivity > 2 && - !bitmap_is_clear_all(used_fields)) + !bitmap_is_clear_all(used_fields) && + thd->variables.use_stat_tables >0) { PARAM param; MEM_ROOT alloc; diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 7ec742b70db..4a7b0df0caf 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -383,7 +383,22 @@ class Column_statistics bool min_max_values_are_provided() { return !is_null(COLUMN_STAT_MIN_VALUE) && - !is_null(COLUMN_STAT_MIN_VALUE); + !is_null(COLUMN_STAT_MAX_VALUE); + } + + /* + This function checks whether the values for the fields of the statistical + tables that were NULL by DEFAULT for a column have changed or not. + + @retval + TRUE: Statistics are not present for a column + FALSE: Statisitics are present for a column + */ + bool check_all_values_are_default() + { + return !(column_stat_nulls ^ + ((1 << (COLUMN_STAT_HISTOGRAM-COLUMN_STAT_COLUMN_NAME))-1) << + (COLUMN_STAT_COLUMN_NAME+1)); } };

2 1

[Commits] 095de15d753: MDEV-17062: Test failure on galera.MW-336
by jan 27 Aug '18

27 Aug '18

revision-id: 095de15d753807ee78e97c2a1ccfc688abb05ee7 (mariadb-10.3.7-144-g095de15d753) parent(s): 2c76653849c1729bedc213b233089bf361c82e12 author: Jan Lindström committer: Jan Lindström timestamp: 2018-08-27 12:03:02 +0300 message: MDEV-17062: Test failure on galera.MW-336 MDEV-17058: Test failure on wsrep.variables MDEV-17060: Test failure on galera.galera_var_slave_threads Fix incorrect calculation of increased applier (slave) threads. Note that increase change takes effect "immediately" but we should use proper wait condition to wait it. Reducing the number of slave threads is not immediate as thread will only exit after a replication event. --- mysql-test/suite/galera/r/MW-336.result | 77 ++++++++++++++++++++++++--------- mysql-test/suite/galera/t/MW-336.test | 54 ++++++++++++----------- mysql-test/suite/wsrep/t/variables.test | 9 +++- sql/wsrep_var.cc | 4 +- 4 files changed, 95 insertions(+), 49 deletions(-) diff --git a/mysql-test/suite/galera/r/MW-336.result b/mysql-test/suite/galera/r/MW-336.result index 0bf8d9d3909..fa5514dfc47 100644 --- a/mysql-test/suite/galera/r/MW-336.result +++ b/mysql-test/suite/galera/r/MW-336.result @@ -2,39 +2,76 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; connection node_1; SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 1; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +COUNT(*) +11 +SHOW STATUS LIKE 'wsrep_thread_count'; +Variable_name Value +wsrep_thread_count 11 connection node_2; INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); connection node_1; SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 20; SET GLOBAL wsrep_slave_threads = 1; connection node_2; INSERT INTO t1 VALUES (1); -INSERT INTO t1 VALUES (2); -INSERT INTO t1 VALUES (3); -INSERT INTO t1 VALUES (4); -INSERT INTO t1 VALUES (5); -INSERT INTO t1 VALUES (6); -INSERT INTO t1 VALUES (7); -INSERT INTO t1 VALUES (8); -INSERT INTO t1 VALUES (9); -connection node_1; +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 0; Warnings: Warning 1292 Truncated incorrect wsrep_slave_threads value: '0' connection node_2; -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); -INSERT INTO t1 VALUES (20); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); connection node_1; SET GLOBAL wsrep_slave_threads = 1; DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MW-336.test b/mysql-test/suite/galera/t/MW-336.test index 8cd363aa019..7af039e577b 100644 --- a/mysql-test/suite/galera/t/MW-336.test +++ b/mysql-test/suite/galera/t/MW-336.test @@ -8,15 +8,28 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; --connection node_1 + SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 1; + +--let $wait_timeout=600 --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +SHOW STATUS LIKE 'wsrep_thread_count'; + --connection node_2 -INSERT INTO t1 VALUES (1); +# Generate 11 replication events +--let $count = 11 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} --connection node_1 + SET GLOBAL wsrep_slave_threads = 10; --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc @@ -28,36 +41,25 @@ SET GLOBAL wsrep_slave_threads = 20; SET GLOBAL wsrep_slave_threads = 1; --connection node_2 -INSERT INTO t1 VALUES (1); -INSERT INTO t1 VALUES (2); -INSERT INTO t1 VALUES (3); -INSERT INTO t1 VALUES (4); -INSERT INTO t1 VALUES (5); -INSERT INTO t1 VALUES (6); -INSERT INTO t1 VALUES (7); -INSERT INTO t1 VALUES (8); -INSERT INTO t1 VALUES (9); - - ---connection node_1 ---let $wait_condition = SELECT COUNT(*) = 12 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); ---source include/wait_condition.inc +# Generate 21 replication events +--let $count = 21 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 0; --connection node_2 -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); -INSERT INTO t1 VALUES (20); +# Generate 21 replication events +--let $count = 21 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} --connection node_1 --let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 1315f090d5c..40b869422ff 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -101,8 +101,10 @@ SHOW STATUS LIKE 'wsrep_thread_count'; --echo # Setting wsrep_cluster_address triggers the creation of --echo # applier/rollbacker threads. SET GLOBAL wsrep_cluster_address= 'gcomm://'; + --echo # Wait for applier threads to get created. -sleep 3; +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count'; +--source include/wait_condition.inc --replace_regex /.*libgalera_smm.*/libgalera_smm.so/ SELECT @@global.wsrep_provider; @@ -113,8 +115,11 @@ SHOW STATUS LIKE 'wsrep_thread_count'; SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads; SET GLOBAL wsrep_slave_threads= 10; + --echo # Wait for applier threads to get created. -sleep 3; +--let $wait_condition = SELECT VARIABLE_VALUE = 11 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count'; +--source include/wait_condition.inc + SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 7fe120beb8e..aaae487b2e2 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -594,7 +594,9 @@ void wsrep_node_address_init (const char* value) static void wsrep_slave_count_change_update () { - wsrep_slave_count_change += (wsrep_slave_threads - wsrep_prev_slave_threads); + wsrep_slave_count_change = (wsrep_slave_threads - wsrep_prev_slave_threads); + WSREP_DEBUG("Change on slave threads: New %lu old %lu difference %lu", + wsrep_slave_threads, wsrep_prev_slave_threads, wsrep_slave_count_change); wsrep_prev_slave_threads = wsrep_slave_threads; }

1 0

[Commits] 7d8d37c31d9: MDEV-17039: Query plan changes when we use GROUP BY optimization with optimizer_use_condition_selectivity=4
by Varun 26 Aug '18

26 Aug '18

revision-id: 7d8d37c31d9cabaf31576eb3edeae6580ffb16b4 (mariadb-10.0.36-8-g7d8d37c31d9) parent(s): bcc677bb7264db08b22284998706b44c377ed8ec author: Varun Gupta committer: Varun Gupta timestamp: 2018-08-23 16:01:58 +0530 message: MDEV-17039: Query plan changes when we use GROUP BY optimization with optimizer_use_condition_selectivity=4 and use_stat_tables= PREFERABLY Currently the code that calculates selectivity for a table does not take into account the case when we can have GROUP BY optimization (looses index scan). --- mysql-test/r/group_min_max.result | 28 ++++++++++++++++++++++++++++ mysql-test/t/group_min_max.test | 17 +++++++++++++++++ sql/opt_range.cc | 13 ++++++++++++- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/group_min_max.result b/mysql-test/r/group_min_max.result index 25cd4a25279..34e1e256faa 100644 --- a/mysql-test/r/group_min_max.result +++ b/mysql-test/r/group_min_max.result @@ -3733,5 +3733,33 @@ id MIN(a) MAX(a) 4 2001-01-04 2001-01-04 DROP TABLE t1; # +# Query plan changes when we use GROUP BY optimization with optimizer_use_condition_selectivity=4 +# and use_stat_tables= PREFERABLY +# +CREATE TABLE t1 (a INT, b INT,c INT DEFAULT 0, INDEX (a,b)); +INSERT INTO t1 (a, b) VALUES (1,1), (1,2), (1,3), (1,4), (1,5), +(2,2), (2,3), (2,1), (3,1), (4,1), (4,2), (4,3), (4,4), (4,5), (4,6); +set @save_optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@optimizer_use_condition_selectivity=4; +set @@use_stat_tables=PREFERABLY; +explain extended SELECT a FROM t1 AS t1_outer WHERE a IN (SELECT max(b) FROM t1 GROUP BY a); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY <subquery2> ALL distinct_key NULL NULL NULL 8 100.00 +1 PRIMARY t1_outer ref a a 5 <subquery2>.max(b) 2 100.00 Using index +2 MATERIALIZED t1 range NULL a 5 NULL 8 100.00 Using index for group-by +Warnings: +Note 1003 select `test`.`t1_outer`.`a` AS `a` from <materialize> (select max(`test`.`t1`.`b`) from `test`.`t1` group by `test`.`t1`.`a`) join `test`.`t1` `t1_outer` where (`test`.`t1_outer`.`a` = `<subquery2>`.`max(b)`) +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set @@use_stat_tables=@save_use_stat_tables; +explain extended SELECT a FROM t1 AS t1_outer WHERE a IN (SELECT max(b) FROM t1 GROUP BY a); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY <subquery2> ALL distinct_key NULL NULL NULL 8 100.00 +1 PRIMARY t1_outer ref a a 5 <subquery2>.max(b) 2 100.00 Using index +2 MATERIALIZED t1 range NULL a 5 NULL 8 100.00 Using index for group-by +Warnings: +Note 1003 select `test`.`t1_outer`.`a` AS `a` from <materialize> (select max(`test`.`t1`.`b`) from `test`.`t1` group by `test`.`t1`.`a`) join `test`.`t1` `t1_outer` where (`test`.`t1_outer`.`a` = `<subquery2>`.`max(b)`) +drop table t1; +# # End of 10.0 tests # diff --git a/mysql-test/t/group_min_max.test b/mysql-test/t/group_min_max.test index 8c9be0ca8db..72a556c9bc2 100644 --- a/mysql-test/t/group_min_max.test +++ b/mysql-test/t/group_min_max.test @@ -1519,6 +1519,23 @@ ALTER TABLE t1 ADD KEY(id,a); SELECT id,MIN(a),MAX(a) FROM t1 WHERE a>=DATE'2001-01-04' GROUP BY id; DROP TABLE t1; +--echo # +--echo # Query plan changes when we use GROUP BY optimization with optimizer_use_condition_selectivity=4 +--echo # and use_stat_tables= PREFERABLY +--echo # + +CREATE TABLE t1 (a INT, b INT,c INT DEFAULT 0, INDEX (a,b)); +INSERT INTO t1 (a, b) VALUES (1,1), (1,2), (1,3), (1,4), (1,5), +(2,2), (2,3), (2,1), (3,1), (4,1), (4,2), (4,3), (4,4), (4,5), (4,6); +set @save_optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; +set @save_use_stat_tables= @@use_stat_tables; +set @@optimizer_use_condition_selectivity=4; +set @@use_stat_tables=PREFERABLY; +explain extended SELECT a FROM t1 AS t1_outer WHERE a IN (SELECT max(b) FROM t1 GROUP BY a); +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set @@use_stat_tables=@save_use_stat_tables; +explain extended SELECT a FROM t1 AS t1_outer WHERE a IN (SELECT max(b) FROM t1 GROUP BY a); +drop table t1; --echo # --echo # End of 10.0 tests diff --git a/sql/opt_range.cc b/sql/opt_range.cc index f1d84e5c623..734adfbc9d1 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -3501,7 +3501,18 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond) table->cond_selectivity= 1.0; - if (!cond || table_records == 0) + if (table_records == 0) + DBUG_RETURN(FALSE); + + QUICK_SELECT_I *quick; + if ((quick=table->reginfo.join_tab->quick) && + quick->get_type() == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + { + table->cond_selectivity*= (quick->records/table_records); + DBUG_RETURN(FALSE); + } + + if (!cond) DBUG_RETURN(FALSE); if (table->pos_in_table_list->schema_table)

2 1