dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable SecondaryCache::CreateFromString to create sec cache based on the uri for CompressedSecondaryCache (#10132) 

Summary:
Update SecondaryCache::CreateFromString and enable it to create sec cache based on the uri for CompressedSecondaryCache.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10132

Test Plan: Add unit tests.

Reviewed By: anand1976

Differential Revision: D36996997

Pulled By: gitbw95

fbshipit-source-id: 882ad563cff6d38b306a53426ad7e47273f34edc
This commit is contained in:
gitbw95 2022-06-10 12:23:10 -07:00 committed by Facebook GitHub Bot
parent d3a3b02134
commit f4052d13b7
2 changed files with 121 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
cache/cache.cc vendored
View File

@ -34,13 +34,61 @@ static std::unordered_map<std::string, OptionTypeInfo>
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
};
static std::unordered_map<std::string, OptionTypeInfo>
comp_sec_cache_options_type_info = {
{"capacity",
{offsetof(struct CompressedSecondaryCacheOptions, capacity),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"num_shard_bits",
{offsetof(struct CompressedSecondaryCacheOptions, num_shard_bits),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"compression_type",
{offsetof(struct CompressedSecondaryCacheOptions, compression_type),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"compress_format_version",
{offsetof(struct CompressedSecondaryCacheOptions,
compress_format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
};
#endif // ROCKSDB_LITE
Status SecondaryCache::CreateFromString(
const ConfigOptions& config_options, const std::string& value,
std::shared_ptr<SecondaryCache>* result) {
return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
result);
if (value.find("compressed_secondary_cache://") == 0) {
std::string args = value;
args.erase(0, std::strlen("compressed_secondary_cache://"));
Status status;
std::shared_ptr<SecondaryCache> sec_cache;
#ifndef ROCKSDB_LITE
CompressedSecondaryCacheOptions sec_cache_opts;
status = OptionTypeInfo::ParseStruct(config_options, "",
&comp_sec_cache_options_type_info, "",
args, &sec_cache_opts);
if (status.ok()) {
sec_cache = NewCompressedSecondaryCache(sec_cache_opts);
}
#else
(void)config_options;
status = Status::NotSupported(
"Cannot load compressed secondary cache in LITE mode ", args);
#endif //! ROCKSDB_LITE
if (status.ok()) {
result->swap(sec_cache);
}
return status;
} else {
return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
result);
}
}
Status Cache::CreateFromString(const ConfigOptions& config_options,

102
cache/compressed_secondary_cache_test.cc vendored
View File

@ -10,6 +10,8 @@
#include "memory/jemalloc_nodump_allocator.h"
#include "memory/memory_allocator.h"
#include "rocksdb/convenience.h"
#include "rocksdb/secondary_cache.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#include "util/compression.h"
@ -79,37 +81,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
void SetFailCreate(bool fail) { fail_create_ = fail; }
void BasicTest(bool sec_cache_is_compressed, bool use_jemalloc) {
CompressedSecondaryCacheOptions opts;
opts.capacity = 2048;
opts.num_shard_bits = 0;
opts.metadata_charge_policy = kDontChargeCacheMetadata;
if (sec_cache_is_compressed) {
if (!LZ4_Supported()) {
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
opts.compression_type = CompressionType::kNoCompression;
}
} else {
opts.compression_type = CompressionType::kNoCompression;
}
if (use_jemalloc) {
JemallocAllocatorOptions jopts;
std::shared_ptr<MemoryAllocator> allocator;
std::string msg;
if (JemallocNodumpAllocator::IsSupported(&msg)) {
Status s = NewJemallocNodumpAllocator(jopts, &allocator);
if (s.ok()) {
opts.memory_allocator = allocator;
}
} else {
ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
}
}
std::shared_ptr<SecondaryCache> sec_cache =
NewCompressedSecondaryCache(opts);
void BasicTestHelper(std::shared_ptr<SecondaryCache> sec_cache) {
bool is_in_sec_cache{true};
// Lookup an non-existent key.
std::unique_ptr<SecondaryCacheResultHandle> handle0 =
@ -160,6 +132,38 @@ class CompressedSecondaryCacheTest : public testing::Test {
sec_cache.reset();
}
void BasicTest(bool sec_cache_is_compressed, bool use_jemalloc) {
CompressedSecondaryCacheOptions opts;
opts.capacity = 2048;
opts.num_shard_bits = 0;
opts.metadata_charge_policy = kDontChargeCacheMetadata;
if (sec_cache_is_compressed) {
if (!LZ4_Supported()) {
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
opts.compression_type = CompressionType::kNoCompression;
}
} else {
opts.compression_type = CompressionType::kNoCompression;
}
if (use_jemalloc) {
JemallocAllocatorOptions jopts;
std::shared_ptr<MemoryAllocator> allocator;
std::string msg;
if (JemallocNodumpAllocator::IsSupported(&msg)) {
Status s = NewJemallocNodumpAllocator(jopts, &allocator);
if (s.ok()) {
opts.memory_allocator = allocator;
}
} else {
ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
}
}
std::shared_ptr<SecondaryCache> sec_cache =
NewCompressedSecondaryCache(opts);
}
void FailsTest(bool sec_cache_is_compressed) {
CompressedSecondaryCacheOptions secondary_cache_opts;
if (sec_cache_is_compressed) {
@ -547,6 +551,42 @@ TEST_F(CompressedSecondaryCacheTest,
BasicTest(true, true);
}
#ifndef ROCKSDB_LITE
TEST_F(CompressedSecondaryCacheTest, BasicTestFromStringWithNoCompression) {
std::string sec_cache_uri =
"compressed_secondary_cache://"
"capacity=2048;num_shard_bits=0;compression_type=kNoCompression";
std::shared_ptr<SecondaryCache> sec_cache;
Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri,
&sec_cache);
EXPECT_OK(s);
BasicTestHelper(sec_cache);
}
TEST_F(CompressedSecondaryCacheTest, BasicTestFromStringWithCompression) {
std::string sec_cache_uri;
if (LZ4_Supported()) {
sec_cache_uri =
"compressed_secondary_cache://"
"capacity=2048;num_shard_bits=0;compression_type=kLZ4Compression;"
"compress_format_version=2";
} else {
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
sec_cache_uri =
"compressed_secondary_cache://"
"capacity=2048;num_shard_bits=0;compression_type=kNoCompression";
}
std::shared_ptr<SecondaryCache> sec_cache;
Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri,
&sec_cache);
EXPECT_OK(s);
BasicTestHelper(sec_cache);
}
#endif // ROCKSDB_LITE
TEST_F(CompressedSecondaryCacheTest, FailsTestWithNoCompression) {
FailsTest(false);
}