mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-25 22:44:05 +00:00
ef6f025563
Summary: * The plan is for AutoHyperClockCache to be selected when HyperClockCacheOptions::estimated_entry_charge == 0, and in that case to use a new configuration option min_avg_entry_charge for determining an extreme case maximum size for the hash table. For the placeholder, a hack is in place in HyperClockCacheOptions::MakeSharedCache() to make the unit tests happy despite the new options not really making sense with the current implementation. * Mostly updating and refactoring tests to test both the current HCC (internal name FixedHyperClockCache) and a placeholder for the new version (internal name AutoHyperClockCache). * Simplify some existing tests not to depend directly on cache type. * Type-parameterize the shard-level unit tests, which unfortunately requires more syntax like `this->` in places for disambiguation. * Added means of choosing auto_hyper_clock_cache to cache_bench, db_bench, and db_stress, including add to crash test. * Add another templated class BaseHyperClockCache to reduce future copy-paste * Added ReportProblems support to cache_bench * Added a DEBUG-level diagnostic to ReportProblems for the variance in load factor throughout the table, which will become more of a concern with linear hashing to be used in the Auto implementation. Example with current Fixed HCC: ``` 2023/08/10-13:41:41.602450 6ac36 [DEBUG] [che/clock_cache.cc:1507] Slot occupancy stats: Overall 49% (129008/262144), Min/Max/Window = 39%/60%/500, MaxRun{Pos/Neg} = 18/17 ``` In other words, with overall occupancy of 49%, the lowest across any 500 contiguous cells is 39% and highest 60%. Longest run of occupied is 18 and longest run of unoccupied is 17. This seems consistent with random samples from a uniform distribution. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11692 Test Plan: Shouldn't be any meaningful changes yet to production code or to what is tested, but there is temporary redundancy in testing until the new implementation is plugged in. Reviewed By: jowlyzhang Differential Revision: D48247413 Pulled By: pdillinger fbshipit-source-id: 11541f996d97af403c2e43c92fb67ff22dd0b5da
132 lines
4.1 KiB
C++
132 lines
4.1 KiB
C++
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <functional>
|
|
|
|
#include "rocksdb/advanced_cache.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
namespace secondary_cache_test_util {
|
|
|
|
struct TestCreateContext : public Cache::CreateContext {
|
|
void SetFailCreate(bool fail) { fail_create_ = fail; }
|
|
|
|
bool fail_create_ = false;
|
|
};
|
|
|
|
class WithCacheType : public TestCreateContext {
|
|
public:
|
|
WithCacheType() {}
|
|
virtual ~WithCacheType() {}
|
|
|
|
class TestItem {
|
|
public:
|
|
TestItem(const char* buf, size_t size) : buf_(new char[size]), size_(size) {
|
|
memcpy(buf_.get(), buf, size);
|
|
}
|
|
~TestItem() = default;
|
|
|
|
char* Buf() { return buf_.get(); }
|
|
[[nodiscard]] size_t Size() const { return size_; }
|
|
std::string ToString() { return std::string(Buf(), Size()); }
|
|
|
|
private:
|
|
std::unique_ptr<char[]> buf_;
|
|
size_t size_;
|
|
};
|
|
|
|
static constexpr auto kLRU = "lru";
|
|
static constexpr auto kFixedHyperClock = "fixed_hyper_clock";
|
|
static constexpr auto kAutoHyperClock = "auto_hyper_clock";
|
|
|
|
// For options other than capacity
|
|
size_t estimated_value_size_ = 1;
|
|
|
|
virtual const std::string& Type() const = 0;
|
|
|
|
static bool IsHyperClock(const std::string& type) {
|
|
return type == kFixedHyperClock || type == kAutoHyperClock;
|
|
}
|
|
|
|
bool IsHyperClock() const { return IsHyperClock(Type()); }
|
|
|
|
std::shared_ptr<Cache> NewCache(
|
|
size_t capacity,
|
|
std::function<void(ShardedCacheOptions&)> modify_opts_fn = {}) {
|
|
const auto& type = Type();
|
|
if (type == kLRU) {
|
|
LRUCacheOptions lru_opts;
|
|
lru_opts.capacity = capacity;
|
|
lru_opts.hash_seed = 0; // deterministic tests
|
|
if (modify_opts_fn) {
|
|
modify_opts_fn(lru_opts);
|
|
}
|
|
return lru_opts.MakeSharedCache();
|
|
}
|
|
if (IsHyperClock(type)) {
|
|
HyperClockCacheOptions hc_opts{
|
|
capacity, type == kFixedHyperClock ? estimated_value_size_ : 0};
|
|
hc_opts.min_avg_entry_charge =
|
|
std::max(size_t{1}, estimated_value_size_ / 2);
|
|
hc_opts.hash_seed = 0; // deterministic tests
|
|
if (modify_opts_fn) {
|
|
modify_opts_fn(hc_opts);
|
|
}
|
|
return hc_opts.MakeSharedCache();
|
|
}
|
|
assert(false);
|
|
return nullptr;
|
|
}
|
|
|
|
std::shared_ptr<Cache> NewCache(
|
|
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
|
CacheMetadataChargePolicy charge_policy = kDontChargeCacheMetadata) {
|
|
return NewCache(capacity, [=](ShardedCacheOptions& opts) {
|
|
opts.num_shard_bits = num_shard_bits;
|
|
opts.strict_capacity_limit = strict_capacity_limit;
|
|
opts.metadata_charge_policy = charge_policy;
|
|
});
|
|
}
|
|
|
|
std::shared_ptr<Cache> NewCache(
|
|
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
|
std::shared_ptr<SecondaryCache> secondary_cache) {
|
|
return NewCache(capacity, [=](ShardedCacheOptions& opts) {
|
|
opts.num_shard_bits = num_shard_bits;
|
|
opts.strict_capacity_limit = strict_capacity_limit;
|
|
opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
|
opts.secondary_cache = secondary_cache;
|
|
});
|
|
}
|
|
|
|
static const Cache::CacheItemHelper* GetHelper(
|
|
CacheEntryRole r = CacheEntryRole::kDataBlock,
|
|
bool secondary_compatible = true, bool fail = false);
|
|
|
|
static const Cache::CacheItemHelper* GetHelperFail(
|
|
CacheEntryRole r = CacheEntryRole::kDataBlock);
|
|
};
|
|
|
|
class WithCacheTypeParam : public WithCacheType,
|
|
public testing::WithParamInterface<std::string> {
|
|
const std::string& Type() const override { return GetParam(); }
|
|
};
|
|
|
|
constexpr auto kLRU = WithCacheType::kLRU;
|
|
constexpr auto kFixedHyperClock = WithCacheType::kFixedHyperClock;
|
|
constexpr auto kAutoHyperClock = WithCacheType::kAutoHyperClock;
|
|
|
|
inline auto GetTestingCacheTypes() {
|
|
return testing::Values(std::string(kLRU), std::string(kFixedHyperClock),
|
|
std::string(kAutoHyperClock));
|
|
}
|
|
|
|
} // namespace secondary_cache_test_util
|
|
} // namespace ROCKSDB_NAMESPACE
|