revision-id: 5349b6d4a81fbc4661c83d722e55159ee9d39ee1 () parent(s): d5b69eb25461da5ff82bf0598aeb5f7f85362bb1 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-01-30 17:27:07 +0300 message: Import the patch from https://github.com/rockset/rocksdb-cloud/issues/45 Will need further adjustments. --- CMakeLists.txt | 55 ++++++++++++++++++++++++++++++- cloud/aws/aws_env.cc | 53 ++++++++++++++++++++++++++--- include/rocksdb/cloud/cloud_env_options.h | 44 +++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1fdf913..df5db6bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,47 @@ option(WITH_SNAPPY "build with SNAPPY" OFF) option(WITH_LZ4 "build with lz4" OFF) option(WITH_ZLIB "build with zlib" OFF) option(WITH_ZSTD "build with zstd" OFF) + + +# find_package(snappy REQUIRED) +# add_definitions(-DSNAPPY) +# include_directories(${SNAPPY_INCLUDE_DIR}) +# list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES}) + +#find_package(rdkafka REQUIRED) +#add_definitions(-DUSE_KAFKA) +#include_directories(${RDKAFKA_INCLUDE_DIR}) +#list(APPEND THIRDPARTY_LIBS ${RDKAFKA_LIBRARIES}) +#CMAKE_LINK_LIBRARY_FLAG +# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lrdkafka++") + +# [MS] Adding debug info +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -g") + +# [MS] Adding librdkafka++ +# rdkafka uses pkgconfig instead of Findrdkafka.cmake +# https://stackoverflow.com/questions/29191855/what-is-the-proper-way-to-use-p... +add_definitions(-DUSE_KAFKA) +# Using system installed rdkafka +# find_package(PkgConfig REQUIRED) +# pkg_check_modules(RDKAFKA REQUIRED rdkafka++) +# include_directories(${RDKAFKA_INCLUDE_DIR}) +# list(APPEND THIRDPARTY_LIBS ${RDKAFKA_LIBRARIES}) +# use custom location for rdkafka +include_directories(/home/myrocks/Work/librdkafka/librdkafka-0.11.5/install/usr/local/include) +list(APPEND THIRDPARTY_LIBS rdkafka rdkafka++) +link_directories(/home/myrocks/Work/librdkafka/librdkafka-0.11.5/install/usr/local/lib) + +# [MS] Adding AWS libs +add_definitions(-DUSE_AWS) +include_directories(/home/myrocks/Work/aws/aws-sdk-cpp_master_2018-06-25/install/usr/local/include) +# -laws-cpp-sdk-s3 -laws-cpp-sdk-kinesis -laws-cpp-sdk-core +list(APPEND THIRDPARTY_LIBS aws-cpp-sdk-s3 aws-cpp-sdk-kinesis aws-cpp-sdk-core) +link_directories(/home/myrocks/Work/aws/aws-sdk-cpp_master_2018-06-25/install/usr/local/lib) + + if(MSVC) # Defaults currently different for GFLAGS. # We will address find_package work a little later @@ -677,6 +718,18 @@ set(SOURCES utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc + cloud/aws/aws_env.cc + cloud/aws/aws_kafka.cc + cloud/aws/aws_kinesis.cc + cloud/aws/aws_log.cc + cloud/aws/aws_retry.cc + cloud/aws/aws_s3.cc + cloud/db_cloud_impl.cc + cloud/cloud_env.cc + cloud/cloud_env_options.cc + cloud/manifest_reader.cc + cloud/purge.cc + cloud/cloud_manifest.cc $<TARGET_OBJECTS:build_version>) if(HAVE_SSE42 AND NOT FORCE_SSE42) @@ -1026,7 +1079,7 @@ if(WITH_TESTS) ) # Tests are excluded from Release builds - set(TEST_EXES ${TESTS}) +set(TEST_EXES ${TESTS}) foreach(sourcefile ${TEST_EXES}) get_filename_component(exename ${sourcefile} NAME_WE) diff --git a/cloud/aws/aws_env.cc b/cloud/aws/aws_env.cc index 7290561d..1c3bf19e 100644 --- a/cloud/aws/aws_env.cc +++ b/cloud/aws/aws_env.cc @@ -278,11 +278,56 @@ AwsEnv::AwsEnv(Env* underlying_env, const std::string& src_bucket_prefix, creds ? "[given]" : "[not given]"); base_env_ = underlying_env; - Aws::InitAPI(Aws::SDKOptions()); - // create AWS S3 client with appropriate timeouts + + // Get additional data for 'sdk_options' and 'config' from cloud_env_options.aws_options + // Check if the corresponded value is not usable - use defaults + + Aws::SDKOptions sdk_options; Aws::Client::ClientConfiguration config; - config.connectTimeoutMs = 30000; - config.requestTimeoutMs = 600000; + + // General SDK options: logging + if (cloud_env_options.aws_options.logLevel != -1) + sdk_options.loggingOptions.logLevel = (Aws::Utils::Logging::LogLevel)cloud_env_options.aws_options.logLevel; + Aws::InitAPI(sdk_options); + + // Timeout control + if (cloud_env_options.aws_options.connectTimeoutMs != -1) + config.connectTimeoutMs = cloud_env_options.aws_options.connectTimeoutMs; + else + config.connectTimeoutMs = 30000; + + if (cloud_env_options.aws_options.requestTimeoutMs != -1) + config.requestTimeoutMs = cloud_env_options.aws_options.requestTimeoutMs; + else + config.requestTimeoutMs = 600000; + + // Endpoint override + if (cloud_env_options.aws_options.endpointOverride != "") + config.endpointOverride = cloud_env_options.aws_options.endpointOverride.c_str(); + + if (cloud_env_options.aws_options.scheme != "") + config.scheme = Aws::Http::SchemeMapper::FromString(cloud_env_options.aws_options.scheme.c_str()); + + if (cloud_env_options.aws_options.verifySSL != -1) + config.verifySSL = (bool)cloud_env_options.aws_options.verifySSL; + + // Proxy settings, not using by default and if proxyHost == "" + if (cloud_env_options.aws_options.proxyHost != "") + { + config.proxyHost = cloud_env_options.aws_options.proxyHost.c_str(); + + if (cloud_env_options.aws_options.proxyScheme != "") + config.proxyScheme = Aws::Http::SchemeMapper::FromString(cloud_env_options.aws_options.proxyScheme.c_str()); + + if (cloud_env_options.aws_options.proxyPort > 0) + config.proxyPort = cloud_env_options.aws_options.proxyPort; + + if (cloud_env_options.aws_options.proxyUserName != "") + config.proxyUserName = cloud_env_options.aws_options.proxyUserName.c_str(); + + if (cloud_env_options.aws_options.proxyPassword != "") + config.proxyPassword = cloud_env_options.aws_options.proxyPassword.c_str(); + } // Setup how retries need to be done config.retryStrategy = diff --git a/include/rocksdb/cloud/cloud_env_options.h b/include/rocksdb/cloud/cloud_env_options.h index af7278fd..57a9bfed 100644 --- a/include/rocksdb/cloud/cloud_env_options.h +++ b/include/rocksdb/cloud/cloud_env_options.h @@ -46,6 +46,47 @@ class KafkaLogOptions { std::unordered_map<std::string, std::string> client_config_params; }; +// Extra data to fill-in Aws::SDKOptions and Aws::Client::ClientConfiguration +// inside AwsEnv::AwsEnv() +// Not usable values, using defaults instead: +// -1 for int/long +// "" for std::string +class AwsOptions { + public: + // General SDK options: logging + int logLevel; // see https://sdk.amazonaws.com/cpp/api/0.14.3/aws-cpp-sdk-core_2include_2aws_2cor... + + // Timeout control + long requestTimeoutMs; + long connectTimeoutMs; + + // Endpoint override + std::string endpointOverride; // local S3/Minio server "127.0.0.1:9000" + std::string scheme; // "http" or "https" + int verifySSL; // 0 - don't verify, 1 - do verify + + // Proxy settings, not using by default and if proxyHost == "" + std::string proxyScheme; + std::string proxyHost; + unsigned proxyPort; + std::string proxyUserName; + std::string proxyPassword; + + // Setting not usable values as defaults (librocksdb/AWS defaults will be used) + AwsOptions() : + logLevel(-1), + requestTimeoutMs(-1), + connectTimeoutMs(-1), + endpointOverride(""), + scheme(""), + verifySSL(-1), + proxyScheme(""), + proxyHost(""), + proxyPort(0), + proxyUserName(""), + proxyPassword("") {} +}; + enum class CloudRequestOpType { kReadOp, kWriteOp, @@ -77,6 +118,9 @@ class CloudEnvOptions { // Only used if keep_local_log_files is true and log_type is kKafka. KafkaLogOptions kafka_log_options; + // To specify AWS related extras: server IP:port, scheme, timeouts, proxy. + AwsOptions aws_options; + // // If true, then sst files are stored locally and uploaded to the cloud in // the background. On restart, all files from the cloud that are not present