diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fa4ea6c29..7a0000ac4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -649,6 +649,7 @@ set(SOURCES cache/cache.cc cache/cache_entry_roles.cc cache/cache_key.cc + cache/cache_helpers.cc cache/cache_reservation_manager.cc cache/charged_cache.cc cache/clock_cache.cc @@ -806,6 +807,7 @@ set(SOURCES table/block_based/block_based_table_iterator.cc table/block_based/block_based_table_reader.cc table/block_based/block_builder.cc + table/block_based/block_cache.cc table/block_based/block_prefetcher.cc table/block_based/block_prefix_index.cc table/block_based/data_block_hash_index.cc diff --git a/HISTORY.md b/HISTORY.md index c0d3515500..be8222cd33 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -19,10 +19,11 @@ ### New Features * When an SstPartitionerFactory is configured, CompactRange() now automatically selects for compaction any files overlapping a partition boundary that is in the compaction range, even if no actual entries are in the requested compaction range. With this feature, manual compaction can be used to (re-)establish SST partition points when SstPartitioner changes, without a full compaction. - -### New Features * Add BackupEngine feature to exclude files from backup that are known to be backed up elsewhere, using `CreateBackupOptions::exclude_files_callback`. To restore the DB, the excluded files must be provided in alternative backup directories using `RestoreOptions::alternate_dirs`. +### Public API Changes +* Substantial changes have been made to the Cache class to support internal development goals. Direct use of Cache class members is discouraged and further breaking modifications are expected in the future. SecondaryCache has some related changes and implementations will need to be updated. (Unlike Cache, SecondaryCache is still intended to support user implementations, and disruptive changes will be avoided.) (#10975) + ## 7.9.0 (11/21/2022) ### Performance Improvements * Fixed an iterator performance regression for delete range users when scanning through a consecutive sequence of range tombstones (#10877). diff --git a/TARGETS b/TARGETS index 32161166f6..e5f887d499 100644 --- a/TARGETS +++ b/TARGETS @@ -11,6 +11,7 @@ load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrap cpp_library_wrapper(name="rocksdb_lib", srcs=[ "cache/cache.cc", "cache/cache_entry_roles.cc", + "cache/cache_helpers.cc", "cache/cache_key.cc", "cache/cache_reservation_manager.cc", "cache/charged_cache.cc", @@ -180,6 +181,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[ "table/block_based/block_based_table_iterator.cc", "table/block_based/block_based_table_reader.cc", "table/block_based/block_builder.cc", + "table/block_based/block_cache.cc", "table/block_based/block_prefetcher.cc", "table/block_based/block_prefix_index.cc", "table/block_based/data_block_footer.cc", @@ -352,6 +354,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[ "cache/cache.cc", "cache/cache_entry_roles.cc", + "cache/cache_helpers.cc", "cache/cache_key.cc", "cache/cache_reservation_manager.cc", "cache/charged_cache.cc", @@ -521,6 +524,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[ "table/block_based/block_based_table_iterator.cc", "table/block_based/block_based_table_reader.cc", "table/block_based/block_builder.cc", + "table/block_based/block_cache.cc", "table/block_based/block_prefetcher.cc", "table/block_based/block_prefix_index.cc", "table/block_based/data_block_footer.cc", diff --git a/cache/cache_bench_tool.cc b/cache/cache_bench_tool.cc index 73360f414d..1dfbfe3c7f 100644 --- a/cache/cache_bench_tool.cc +++ b/cache/cache_bench_tool.cc @@ -226,7 +226,7 @@ struct KeyGen { } }; -char* createValue(Random64& rnd) { +Cache::ObjectPtr createValue(Random64& rnd) { char* rv = new char[FLAGS_value_bytes]; // Fill with some filler data, and take some CPU time for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) { @@ -236,28 +236,33 @@ char* createValue(Random64& rnd) { } // Callbacks for secondary cache -size_t SizeFn(void* /*obj*/) { return FLAGS_value_bytes; } +size_t SizeFn(Cache::ObjectPtr /*obj*/) { return FLAGS_value_bytes; } -Status SaveToFn(void* obj, size_t /*offset*/, size_t size, void* out) { - memcpy(out, obj, size); +Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/, + size_t length, char* out) { + memcpy(out, from_obj, length); return Status::OK(); } -// Different deleters to simulate using deleter to gather -// stats on the code origin and kind of cache entries. -void deleter1(const Slice& /*key*/, void* value) { - delete[] static_cast(value); -} -void deleter2(const Slice& /*key*/, void* value) { - delete[] static_cast(value); -} -void deleter3(const Slice& /*key*/, void* value) { +Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/, + MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj, + size_t* out_charge) { + *out_obj = new char[data.size()]; + memcpy(*out_obj, data.data(), data.size()); + *out_charge = data.size(); + return Status::OK(); +}; + +void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { delete[] static_cast(value); } -Cache::CacheItemHelper helper1(SizeFn, SaveToFn, deleter1); -Cache::CacheItemHelper helper2(SizeFn, SaveToFn, deleter2); -Cache::CacheItemHelper helper3(SizeFn, SaveToFn, deleter3); +Cache::CacheItemHelper helper1(CacheEntryRole::kDataBlock, DeleteFn, SizeFn, + SaveToFn, CreateFn); +Cache::CacheItemHelper helper2(CacheEntryRole::kIndexBlock, DeleteFn, SizeFn, + SaveToFn, CreateFn); +Cache::CacheItemHelper helper3(CacheEntryRole::kFilterBlock, DeleteFn, SizeFn, + SaveToFn, CreateFn); } // namespace class CacheBench { @@ -436,7 +441,7 @@ class CacheBench { uint64_t total_entry_count = 0; uint64_t table_occupancy = 0; uint64_t table_size = 0; - std::set deleters; + std::set helpers; StopWatchNano timer(clock); for (;;) { @@ -461,7 +466,7 @@ class CacheBench { << BytesToHumanString(static_cast( 1.0 * total_charge / total_entry_count)) << "\n" - << "Unique deleters: " << deleters.size() << "\n"; + << "Unique helpers: " << helpers.size() << "\n"; *stats_report = ostr.str(); return; } @@ -477,14 +482,14 @@ class CacheBench { total_key_size = 0; total_charge = 0; total_entry_count = 0; - deleters.clear(); - auto fn = [&](const Slice& key, void* /*value*/, size_t charge, - Cache::DeleterFn deleter) { + helpers.clear(); + auto fn = [&](const Slice& key, Cache::ObjectPtr /*value*/, size_t charge, + const Cache::CacheItemHelper* helper) { total_key_size += key.size(); total_charge += charge; ++total_entry_count; - // Something slightly more expensive as in (future) stats by category - deleters.insert(deleter); + // Something slightly more expensive as in stats by category + helpers.insert(helper); }; timer.Start(); Cache::ApplyToAllEntriesOptions opts; @@ -533,14 +538,6 @@ class CacheBench { for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) { Slice key = gen.GetRand(thread->rnd, max_key_, max_log_); uint64_t random_op = thread->rnd.Next(); - Cache::CreateCallback create_cb = [](const void* buf, size_t size, - void** out_obj, - size_t* charge) -> Status { - *out_obj = reinterpret_cast(new char[size]); - memcpy(*out_obj, buf, size); - *charge = size; - return Status::OK(); - }; timer.Start(); @@ -550,8 +547,8 @@ class CacheBench { handle = nullptr; } // do lookup - handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW, - true); + handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, + Cache::Priority::LOW, true); if (handle) { if (!FLAGS_lean) { // do something with the data @@ -579,8 +576,8 @@ class CacheBench { handle = nullptr; } // do lookup - handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW, - true); + handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, + Cache::Priority::LOW, true); if (handle) { if (!FLAGS_lean) { // do something with the data diff --git a/cache/cache_entry_roles.cc b/cache/cache_entry_roles.cc index b27349554d..f83ada2313 100644 --- a/cache/cache_entry_roles.cc +++ b/cache/cache_entry_roles.cc @@ -101,34 +101,4 @@ std::string BlockCacheEntryStatsMapKeys::UsedPercent(CacheEntryRole role) { return GetPrefixedCacheEntryRoleName(kPrefix, role); } -namespace { - -struct Registry { - std::mutex mutex; - UnorderedMap role_map; - void Register(Cache::DeleterFn fn, CacheEntryRole role) { - std::lock_guard lock(mutex); - role_map[fn] = role; - } - UnorderedMap Copy() { - std::lock_guard lock(mutex); - return role_map; - } -}; - -Registry& GetRegistry() { - STATIC_AVOID_DESTRUCTION(Registry, registry); - return registry; -} - -} // namespace - -void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role) { - GetRegistry().Register(fn, role); -} - -UnorderedMap CopyCacheDeleterRoleMap() { - return GetRegistry().Copy(); -} - } // namespace ROCKSDB_NAMESPACE diff --git a/cache/cache_entry_roles.h b/cache/cache_entry_roles.h index 5a49fdfd4b..78bec792f2 100644 --- a/cache/cache_entry_roles.h +++ b/cache/cache_entry_roles.h @@ -7,11 +7,8 @@ #include #include -#include -#include #include "rocksdb/cache.h" -#include "util/hash_containers.h" namespace ROCKSDB_NAMESPACE { @@ -20,84 +17,4 @@ extern std::array extern std::array kCacheEntryRoleToHyphenString; -// To associate cache entries with their role, we use a hack on the -// existing Cache interface. Because the deleter of an entry can authenticate -// the code origin of an entry, we can elaborate the choice of deleter to -// also encode role information, without inferring false role information -// from entries not choosing to encode a role. -// -// The rest of this file is for handling mappings between deleters and -// roles. - -// To infer a role from a deleter, the deleter must be registered. This -// can be done "manually" with this function. This function is thread-safe, -// and the registration mappings go into private but static storage. (Note -// that DeleterFn is a function pointer, not std::function. Registrations -// should not be too many.) -void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role); - -// Gets a copy of the registered deleter -> role mappings. This is the only -// function for reading the mappings made with RegisterCacheDeleterRole. -// Why only this interface for reading? -// * This function has to be thread safe, which could incur substantial -// overhead. We should not pay this overhead for every deleter look-up. -// * This is suitable for preparing for batch operations, like with -// CacheEntryStatsCollector. -// * The number of mappings should be sufficiently small (dozens). -UnorderedMap CopyCacheDeleterRoleMap(); - -// ************************************************************** // -// An automatic registration infrastructure. This enables code -// to simply ask for a deleter associated with a particular type -// and role, and registration is automatic. In a sense, this is -// a small dependency injection infrastructure, because linking -// in new deleter instantiations is essentially sufficient for -// making stats collection (using CopyCacheDeleterRoleMap) aware -// of them. - -namespace cache_entry_roles_detail { - -template -struct RegisteredDeleter { - RegisteredDeleter() { RegisterCacheDeleterRole(Delete, R); } - - // These have global linkage to help ensure compiler optimizations do not - // break uniqueness for each - static void Delete(const Slice& /* key */, void* value) { - // Supports T == Something[], unlike delete operator - std::default_delete()( - static_cast::type*>(value)); - } -}; - -template -struct RegisteredNoopDeleter { - RegisteredNoopDeleter() { RegisterCacheDeleterRole(Delete, R); } - - static void Delete(const Slice& /* key */, void* /* value */) { - // Here was `assert(value == nullptr);` but we can also put pointers - // to static data in Cache, for testing at least. - } -}; - -} // namespace cache_entry_roles_detail - -// Get an automatically registered deleter for value type T and role R. -// Based on C++ semantics, registration is invoked exactly once in a -// thread-safe way on first call to this function, for each . -template -Cache::DeleterFn GetCacheEntryDeleterForRole() { - static cache_entry_roles_detail::RegisteredDeleter reg; - return reg.Delete; -} - -// Get an automatically registered no-op deleter (value should be nullptr) -// and associated with role R. This is used for Cache "reservation" entries -// such as for WriteBufferManager. -template -Cache::DeleterFn GetNoopDeleterForRole() { - static cache_entry_roles_detail::RegisteredNoopDeleter reg; - return reg.Delete; -} - } // namespace ROCKSDB_NAMESPACE diff --git a/cache/cache_entry_stats.h b/cache/cache_entry_stats.h index 63b12735b9..054304086d 100644 --- a/cache/cache_entry_stats.h +++ b/cache/cache_entry_stats.h @@ -10,8 +10,8 @@ #include #include -#include "cache/cache_helpers.h" #include "cache/cache_key.h" +#include "cache/typed_cache.h" #include "port/lang.h" #include "rocksdb/cache.h" #include "rocksdb/status.h" @@ -111,11 +111,14 @@ class CacheEntryStatsCollector { // Gets or creates a shared instance of CacheEntryStatsCollector in the // cache itself, and saves into `ptr`. This shared_ptr will hold the // entry in cache until all refs are destroyed. - static Status GetShared(Cache *cache, SystemClock *clock, + static Status GetShared(Cache *raw_cache, SystemClock *clock, std::shared_ptr *ptr) { - const Slice &cache_key = GetCacheKey(); + assert(raw_cache); + BasicTypedCacheInterface + cache{raw_cache}; - Cache::Handle *h = cache->Lookup(cache_key); + const Slice &cache_key = GetCacheKey(); + auto h = cache.Lookup(cache_key); if (h == nullptr) { // Not yet in cache, but Cache doesn't provide a built-in way to // avoid racing insert. So we double-check under a shared mutex, @@ -123,15 +126,15 @@ class CacheEntryStatsCollector { STATIC_AVOID_DESTRUCTION(std::mutex, static_mutex); std::lock_guard lock(static_mutex); - h = cache->Lookup(cache_key); + h = cache.Lookup(cache_key); if (h == nullptr) { - auto new_ptr = new CacheEntryStatsCollector(cache, clock); + auto new_ptr = new CacheEntryStatsCollector(cache.get(), clock); // TODO: non-zero charge causes some tests that count block cache // usage to go flaky. Fix the problem somehow so we can use an // accurate charge. size_t charge = 0; - Status s = cache->Insert(cache_key, new_ptr, charge, Deleter, &h, - Cache::Priority::HIGH); + Status s = + cache.Insert(cache_key, new_ptr, charge, &h, Cache::Priority::HIGH); if (!s.ok()) { assert(h == nullptr); delete new_ptr; @@ -140,11 +143,11 @@ class CacheEntryStatsCollector { } } // If we reach here, shared entry is in cache with handle `h`. - assert(cache->GetDeleter(h) == Deleter); + assert(cache.get()->GetCacheItemHelper(h) == &cache.kBasicHelper); // Build an aliasing shared_ptr that keeps `ptr` in cache while there // are references. - *ptr = MakeSharedCacheHandleGuard(cache, h); + *ptr = cache.SharedGuard(h); return Status::OK(); } @@ -157,10 +160,6 @@ class CacheEntryStatsCollector { cache_(cache), clock_(clock) {} - static void Deleter(const Slice &, void *value) { - delete static_cast(value); - } - static const Slice &GetCacheKey() { // For each template instantiation static CacheKey ckey = CacheKey::CreateUniqueForProcessLifetime(); diff --git a/cache/cache_helpers.cc b/cache/cache_helpers.cc new file mode 100644 index 0000000000..22597bf6da --- /dev/null +++ b/cache/cache_helpers.cc @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/cache_helpers.h" + +namespace ROCKSDB_NAMESPACE { + +void ReleaseCacheHandleCleanup(void* arg1, void* arg2) { + Cache* const cache = static_cast(arg1); + assert(cache); + + Cache::Handle* const cache_handle = static_cast(arg2); + assert(cache_handle); + + cache->Release(cache_handle); +} + +Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved, + Cache::CreateContext* create_context, + const Cache::CacheItemHelper* helper, + Cache::Priority priority, size_t* out_charge) { + assert(helper); + assert(helper->create_cb); + Cache::ObjectPtr value; + size_t charge; + Status st = helper->create_cb(saved, create_context, + cache->memory_allocator(), &value, &charge); + if (st.ok()) { + st = + cache->Insert(key, value, helper, charge, /*handle*/ nullptr, priority); + if (out_charge) { + *out_charge = charge; + } + } + return st; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/cache/cache_helpers.h b/cache/cache_helpers.h index 7ea2365b88..eb4559dfe9 100644 --- a/cache/cache_helpers.h +++ b/cache/cache_helpers.h @@ -17,22 +17,17 @@ template T* GetFromCacheHandle(Cache* cache, Cache::Handle* handle) { assert(cache); assert(handle); - return static_cast(cache->Value(handle)); } -// Simple generic deleter for Cache (to be used with Cache::Insert). -template -void DeleteCacheEntry(const Slice& /* key */, void* value) { - delete static_cast(value); -} - // Turns a T* into a Slice so it can be used as a key with Cache. template -Slice GetSlice(const T* t) { +Slice GetSliceForKey(const T* t) { return Slice(reinterpret_cast(t), sizeof(T)); } +void ReleaseCacheHandleCleanup(void* arg1, void* arg2); + // Generic resource management object for cache handles that releases the handle // when destroyed. Has unique ownership of the handle, so copying it is not // allowed, while moving it transfers ownership. @@ -88,7 +83,7 @@ class CacheHandleGuard { if (cleanable) { if (handle_ != nullptr) { assert(cache_); - cleanable->RegisterCleanup(&ReleaseCacheHandle, cache_, handle_); + cleanable->RegisterCleanup(&ReleaseCacheHandleCleanup, cache_, handle_); } } ResetFields(); @@ -115,16 +110,6 @@ class CacheHandleGuard { value_ = nullptr; } - static void ReleaseCacheHandle(void* arg1, void* arg2) { - Cache* const cache = static_cast(arg1); - assert(cache); - - Cache::Handle* const cache_handle = static_cast(arg2); - assert(cache_handle); - - cache->Release(cache_handle); - } - private: Cache* cache_ = nullptr; Cache::Handle* handle_ = nullptr; @@ -139,7 +124,16 @@ template std::shared_ptr MakeSharedCacheHandleGuard(Cache* cache, Cache::Handle* handle) { auto wrapper = std::make_shared>(cache, handle); - return std::shared_ptr(wrapper, static_cast(cache->Value(handle))); + return std::shared_ptr(wrapper, GetFromCacheHandle(cache, handle)); } +// Given the persistable data (saved) for a block cache entry, parse that +// into a cache entry object and insert it into the given cache. The charge +// of the new entry can be returned to the caller through `out_charge`. +Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved, + Cache::CreateContext* create_context, + const Cache::CacheItemHelper* helper, + Cache::Priority priority = Cache::Priority::LOW, + size_t* out_charge = nullptr); + } // namespace ROCKSDB_NAMESPACE diff --git a/cache/cache_reservation_manager.cc b/cache/cache_reservation_manager.cc index 53dee5d790..b43bfddc69 100644 --- a/cache/cache_reservation_manager.cc +++ b/cache/cache_reservation_manager.cc @@ -13,7 +13,6 @@ #include #include -#include "cache/cache_entry_roles.h" #include "rocksdb/cache.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" @@ -41,17 +40,17 @@ CacheReservationManagerImpl< template CacheReservationManagerImpl::CacheReservationManagerImpl( std::shared_ptr cache, bool delayed_decrease) - : delayed_decrease_(delayed_decrease), + : cache_(cache), + delayed_decrease_(delayed_decrease), cache_allocated_size_(0), memory_used_(0) { assert(cache != nullptr); - cache_ = cache; } template CacheReservationManagerImpl::~CacheReservationManagerImpl() { for (auto* handle : dummy_handles_) { - cache_->Release(handle, true); + cache_.ReleaseAndEraseIfLastRef(handle); } } @@ -115,8 +114,7 @@ Status CacheReservationManagerImpl::IncreaseCacheReservation( Status return_status = Status::OK(); while (new_mem_used > cache_allocated_size_.load(std::memory_order_relaxed)) { Cache::Handle* handle = nullptr; - return_status = cache_->Insert(GetNextCacheKey(), nullptr, kSizeDummyEntry, - GetNoopDeleterForRole(), &handle); + return_status = cache_.Insert(GetNextCacheKey(), kSizeDummyEntry, &handle); if (return_status != Status::OK()) { return return_status; @@ -141,7 +139,7 @@ Status CacheReservationManagerImpl::DecreaseCacheReservation( cache_allocated_size_.load(std::memory_order_relaxed)) { assert(!dummy_handles_.empty()); auto* handle = dummy_handles_.back(); - cache_->Release(handle, true); + cache_.ReleaseAndEraseIfLastRef(handle); dummy_handles_.pop_back(); cache_allocated_size_ -= kSizeDummyEntry; } @@ -169,8 +167,9 @@ Slice CacheReservationManagerImpl::GetNextCacheKey() { } template -Cache::DeleterFn CacheReservationManagerImpl::TEST_GetNoopDeleterForRole() { - return GetNoopDeleterForRole(); +const Cache::CacheItemHelper* +CacheReservationManagerImpl::TEST_GetCacheItemHelperForRole() { + return &CacheInterface::kHelper; } template class CacheReservationManagerImpl< diff --git a/cache/cache_reservation_manager.h b/cache/cache_reservation_manager.h index 147aaa915d..08bf59b006 100644 --- a/cache/cache_reservation_manager.h +++ b/cache/cache_reservation_manager.h @@ -18,7 +18,7 @@ #include "cache/cache_entry_roles.h" #include "cache/cache_key.h" -#include "rocksdb/cache.h" +#include "cache/typed_cache.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" #include "util/coding.h" @@ -197,10 +197,10 @@ class CacheReservationManagerImpl static constexpr std::size_t GetDummyEntrySize() { return kSizeDummyEntry; } - // For testing only - it is to help ensure the NoopDeleterForRole + // For testing only - it is to help ensure the CacheItemHelperForRole // accessed from CacheReservationManagerImpl and the one accessed from the // test are from the same translation units - static Cache::DeleterFn TEST_GetNoopDeleterForRole(); + static const Cache::CacheItemHelper *TEST_GetCacheItemHelperForRole(); private: static constexpr std::size_t kSizeDummyEntry = 256 * 1024; @@ -211,7 +211,8 @@ class CacheReservationManagerImpl Status IncreaseCacheReservation(std::size_t new_mem_used); Status DecreaseCacheReservation(std::size_t new_mem_used); - std::shared_ptr cache_; + using CacheInterface = PlaceholderSharedCacheInterface; + CacheInterface cache_; bool delayed_decrease_; std::atomic cache_allocated_size_; std::size_t memory_used_; diff --git a/cache/cache_test.cc b/cache/cache_test.cc index 212d65d96c..32335f3d2b 100644 --- a/cache/cache_test.cc +++ b/cache/cache_test.cc @@ -16,6 +16,7 @@ #include #include "cache/lru_cache.h" +#include "cache/typed_cache.h" #include "port/stack_trace.h" #include "test_util/testharness.h" #include "util/coding.h" @@ -55,23 +56,31 @@ int DecodeKey32Bits(const Slice& k) { return DecodeFixed32(k.data()); } -void* EncodeValue(uintptr_t v) { return reinterpret_cast(v); } +Cache::ObjectPtr EncodeValue(uintptr_t v) { + return reinterpret_cast(v); +} int DecodeValue(void* v) { return static_cast(reinterpret_cast(v)); } -void DumbDeleter(const Slice& /*key*/, void* /*value*/) {} +const Cache::CacheItemHelper kDumbHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr /*value*/, MemoryAllocator* /*alloc*/) {}}; -void EraseDeleter1(const Slice& /*key*/, void* value) { - Cache* cache = reinterpret_cast(value); - cache->Erase("foo"); -} +const Cache::CacheItemHelper kEraseOnDeleteHelper1{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + Cache* cache = static_cast(value); + cache->Erase("foo"); + }}; -void EraseDeleter2(const Slice& /*key*/, void* value) { - Cache* cache = reinterpret_cast(value); - cache->Erase(EncodeKey16Bytes(1234)); -} +const Cache::CacheItemHelper kEraseOnDeleteHelper2{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + Cache* cache = static_cast(value); + cache->Erase(EncodeKey16Bytes(1234)); + }}; const std::string kLRU = "lru"; const std::string kHyperClock = "hyper_clock"; @@ -83,14 +92,11 @@ class CacheTest : public testing::TestWithParam { static CacheTest* current_; static std::string type_; - static void Deleter(const Slice& key, void* v) { - if (type_ == kHyperClock) { - current_->deleted_keys_.push_back(DecodeKey16Bytes(key)); - } else { - current_->deleted_keys_.push_back(DecodeKey32Bits(key)); - } + static void Deleter(Cache::ObjectPtr v, MemoryAllocator*) { current_->deleted_values_.push_back(DecodeValue(v)); } + static constexpr Cache::CacheItemHelper kHelper{CacheEntryRole::kMisc, + &Deleter}; static const int kCacheSize = 1000; static const int kNumShardBits = 4; @@ -98,7 +104,6 @@ class CacheTest : public testing::TestWithParam { static const int kCacheSize2 = 100; static const int kNumShardBits2 = 2; - std::vector deleted_keys_; std::vector deleted_values_; std::shared_ptr cache_; std::shared_ptr cache2_; @@ -182,8 +187,8 @@ class CacheTest : public testing::TestWithParam { void Insert(std::shared_ptr cache, int key, int value, int charge = 1) { - EXPECT_OK(cache->Insert(EncodeKey(key), EncodeValue(value), charge, - &CacheTest::Deleter)); + EXPECT_OK( + cache->Insert(EncodeKey(key), EncodeValue(value), &kHelper, charge)); } void Erase(std::shared_ptr cache, int key) { @@ -236,10 +241,8 @@ TEST_P(CacheTest, UsageTest) { key = EncodeKey(i); } auto kv_size = key.size() + 5; - ASSERT_OK(cache->Insert(key, reinterpret_cast(value), kv_size, - DumbDeleter)); - ASSERT_OK(precise_cache->Insert(key, reinterpret_cast(value), - kv_size, DumbDeleter)); + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size)); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size)); usage += kv_size; ASSERT_EQ(usage, cache->GetUsage()); if (type == kHyperClock) { @@ -262,10 +265,8 @@ TEST_P(CacheTest, UsageTest) { } else { key = EncodeKey(static_cast(1000 + i)); } - ASSERT_OK(cache->Insert(key, reinterpret_cast(value), key.size() + 5, - DumbDeleter)); - ASSERT_OK(precise_cache->Insert(key, reinterpret_cast(value), - key.size() + 5, DumbDeleter)); + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5)); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5)); } // the usage should be close to the capacity @@ -320,11 +321,9 @@ TEST_P(CacheTest, PinnedUsageTest) { auto kv_size = key.size() + 5; Cache::Handle* handle; Cache::Handle* handle_in_precise_cache; - ASSERT_OK(cache->Insert(key, reinterpret_cast(value), kv_size, - DumbDeleter, &handle)); + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size, &handle)); assert(handle); - ASSERT_OK(precise_cache->Insert(key, reinterpret_cast(value), - kv_size, DumbDeleter, + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size, &handle_in_precise_cache)); assert(handle_in_precise_cache); pinned_usage += kv_size; @@ -365,10 +364,8 @@ TEST_P(CacheTest, PinnedUsageTest) { } else { key = EncodeKey(static_cast(1000 + i)); } - ASSERT_OK(cache->Insert(key, reinterpret_cast(value), key.size() + 5, - DumbDeleter)); - ASSERT_OK(precise_cache->Insert(key, reinterpret_cast(value), - key.size() + 5, DumbDeleter)); + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5)); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5)); } ASSERT_EQ(pinned_usage, cache->GetPinnedUsage()); ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage()); @@ -416,8 +413,7 @@ TEST_P(CacheTest, HitAndMiss) { ASSERT_EQ(201, Lookup(200)); ASSERT_EQ(-1, Lookup(300)); - ASSERT_EQ(1U, deleted_keys_.size()); - ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(1U, deleted_values_.size()); if (GetParam() == kHyperClock) { ASSERT_EQ(102, deleted_values_[0]); } else { @@ -438,21 +434,20 @@ TEST_P(CacheTest, InsertSameKey) { TEST_P(CacheTest, Erase) { Erase(200); - ASSERT_EQ(0U, deleted_keys_.size()); + ASSERT_EQ(0U, deleted_values_.size()); Insert(100, 101); Insert(200, 201); Erase(100); ASSERT_EQ(-1, Lookup(100)); ASSERT_EQ(201, Lookup(200)); - ASSERT_EQ(1U, deleted_keys_.size()); - ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(1U, deleted_values_.size()); ASSERT_EQ(101, deleted_values_[0]); Erase(100); ASSERT_EQ(-1, Lookup(100)); ASSERT_EQ(201, Lookup(200)); - ASSERT_EQ(1U, deleted_keys_.size()); + ASSERT_EQ(1U, deleted_values_.size()); } TEST_P(CacheTest, EntriesArePinned) { @@ -469,23 +464,21 @@ TEST_P(CacheTest, EntriesArePinned) { Insert(100, 102); Cache::Handle* h2 = cache_->Lookup(EncodeKey(100)); ASSERT_EQ(102, DecodeValue(cache_->Value(h2))); - ASSERT_EQ(0U, deleted_keys_.size()); + ASSERT_EQ(0U, deleted_values_.size()); ASSERT_EQ(2U, cache_->GetUsage()); cache_->Release(h1); - ASSERT_EQ(1U, deleted_keys_.size()); - ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(1U, deleted_values_.size()); ASSERT_EQ(101, deleted_values_[0]); ASSERT_EQ(1U, cache_->GetUsage()); Erase(100); ASSERT_EQ(-1, Lookup(100)); - ASSERT_EQ(1U, deleted_keys_.size()); + ASSERT_EQ(1U, deleted_values_.size()); ASSERT_EQ(1U, cache_->GetUsage()); cache_->Release(h2); - ASSERT_EQ(2U, deleted_keys_.size()); - ASSERT_EQ(100, deleted_keys_[1]); + ASSERT_EQ(2U, deleted_values_.size()); ASSERT_EQ(102, deleted_values_[1]); ASSERT_EQ(0U, cache_->GetUsage()); } @@ -588,9 +581,9 @@ TEST_P(CacheTest, EvictEmptyCache) { // Insert item large than capacity to trigger eviction on empty cache. auto cache = NewCache(1, 0, false); if (type == kLRU) { - ASSERT_OK(cache->Insert("foo", nullptr, 10, DumbDeleter)); + ASSERT_OK(cache->Insert("foo", nullptr, &kDumbHelper, 10)); } else { - ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, 10, DumbDeleter)); + ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10)); } } @@ -601,19 +594,19 @@ TEST_P(CacheTest, EraseFromDeleter) { // the cache at that point. std::shared_ptr cache = NewCache(10, 0, false); std::string foo, bar; - Cache::DeleterFn erase_deleter; + const Cache::CacheItemHelper* erase_helper; if (type == kLRU) { foo = "foo"; bar = "bar"; - erase_deleter = EraseDeleter1; + erase_helper = &kEraseOnDeleteHelper1; } else { foo = EncodeKey(1234); bar = EncodeKey(5678); - erase_deleter = EraseDeleter2; + erase_helper = &kEraseOnDeleteHelper2; } - ASSERT_OK(cache->Insert(foo, nullptr, 1, DumbDeleter)); - ASSERT_OK(cache->Insert(bar, cache.get(), 1, erase_deleter)); + ASSERT_OK(cache->Insert(foo, nullptr, &kDumbHelper, 1)); + ASSERT_OK(cache->Insert(bar, cache.get(), erase_helper, 1)); cache->Erase(bar); ASSERT_EQ(nullptr, cache->Lookup(foo)); @@ -675,50 +668,51 @@ TEST_P(CacheTest, NewId) { ASSERT_NE(a, b); } -class Value { - public: - explicit Value(int v) : v_(v) {} - - int v_; -}; - -namespace { -void deleter(const Slice& /*key*/, void* value) { - delete static_cast(value); -} -} // namespace - TEST_P(CacheTest, ReleaseAndErase) { std::shared_ptr cache = NewCache(5, 0, false); Cache::Handle* handle; - Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1, - &CacheTest::Deleter, &handle); + Status s = + cache->Insert(EncodeKey(100), EncodeValue(100), &kHelper, 1, &handle); ASSERT_TRUE(s.ok()); ASSERT_EQ(5U, cache->GetCapacity()); ASSERT_EQ(1U, cache->GetUsage()); - ASSERT_EQ(0U, deleted_keys_.size()); + ASSERT_EQ(0U, deleted_values_.size()); auto erased = cache->Release(handle, true); ASSERT_TRUE(erased); // This tests that deleter has been called - ASSERT_EQ(1U, deleted_keys_.size()); + ASSERT_EQ(1U, deleted_values_.size()); } TEST_P(CacheTest, ReleaseWithoutErase) { std::shared_ptr cache = NewCache(5, 0, false); Cache::Handle* handle; - Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1, - &CacheTest::Deleter, &handle); + Status s = + cache->Insert(EncodeKey(100), EncodeValue(100), &kHelper, 1, &handle); ASSERT_TRUE(s.ok()); ASSERT_EQ(5U, cache->GetCapacity()); ASSERT_EQ(1U, cache->GetUsage()); - ASSERT_EQ(0U, deleted_keys_.size()); + ASSERT_EQ(0U, deleted_values_.size()); auto erased = cache->Release(handle); ASSERT_FALSE(erased); // This tests that deleter is not called. When cache has free capacity it is // not expected to immediately erase the released items. - ASSERT_EQ(0U, deleted_keys_.size()); + ASSERT_EQ(0U, deleted_values_.size()); } +namespace { +class Value { + public: + explicit Value(int v) : v_(v) {} + + int v_; + + static constexpr auto kCacheEntryRole = CacheEntryRole::kMisc; +}; + +using SharedCache = BasicTypedSharedCacheInterface; +using TypedHandle = SharedCache::TypedHandle; +} // namespace + TEST_P(CacheTest, SetCapacity) { auto type = GetParam(); if (type == kHyperClock) { @@ -731,19 +725,19 @@ TEST_P(CacheTest, SetCapacity) { // lets create a cache with capacity 5, // then, insert 5 elements, then increase capacity // to 10, returned capacity should be 10, usage=5 - std::shared_ptr cache = NewCache(5, 0, false); - std::vector handles(10); + SharedCache cache{NewCache(5, 0, false)}; + std::vector handles(10); // Insert 5 entries, but not releasing. for (int i = 0; i < 5; i++) { std::string key = EncodeKey(i + 1); - Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]); + Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); ASSERT_TRUE(s.ok()); } - ASSERT_EQ(5U, cache->GetCapacity()); - ASSERT_EQ(5U, cache->GetUsage()); - cache->SetCapacity(10); - ASSERT_EQ(10U, cache->GetCapacity()); - ASSERT_EQ(5U, cache->GetUsage()); + ASSERT_EQ(5U, cache.get()->GetCapacity()); + ASSERT_EQ(5U, cache.get()->GetUsage()); + cache.get()->SetCapacity(10); + ASSERT_EQ(10U, cache.get()->GetCapacity()); + ASSERT_EQ(5U, cache.get()->GetUsage()); // test2: decrease capacity // insert 5 more elements to cache, then release 5, @@ -751,77 +745,77 @@ TEST_P(CacheTest, SetCapacity) { // and usage should be 7 for (int i = 5; i < 10; i++) { std::string key = EncodeKey(i + 1); - Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]); + Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); ASSERT_TRUE(s.ok()); } - ASSERT_EQ(10U, cache->GetCapacity()); - ASSERT_EQ(10U, cache->GetUsage()); + ASSERT_EQ(10U, cache.get()->GetCapacity()); + ASSERT_EQ(10U, cache.get()->GetUsage()); for (int i = 0; i < 5; i++) { - cache->Release(handles[i]); + cache.Release(handles[i]); } - ASSERT_EQ(10U, cache->GetCapacity()); - ASSERT_EQ(10U, cache->GetUsage()); - cache->SetCapacity(7); - ASSERT_EQ(7, cache->GetCapacity()); - ASSERT_EQ(7, cache->GetUsage()); + ASSERT_EQ(10U, cache.get()->GetCapacity()); + ASSERT_EQ(10U, cache.get()->GetUsage()); + cache.get()->SetCapacity(7); + ASSERT_EQ(7, cache.get()->GetCapacity()); + ASSERT_EQ(7, cache.get()->GetUsage()); // release remaining 5 to keep valgrind happy for (int i = 5; i < 10; i++) { - cache->Release(handles[i]); + cache.Release(handles[i]); } // Make sure this doesn't crash or upset ASAN/valgrind - cache->DisownData(); + cache.get()->DisownData(); } TEST_P(LRUCacheTest, SetStrictCapacityLimit) { // test1: set the flag to false. Insert more keys than capacity. See if they // all go through. - std::shared_ptr cache = NewCache(5, 0, false); - std::vector handles(10); + SharedCache cache{NewCache(5, 0, false)}; + std::vector handles(10); Status s; for (int i = 0; i < 10; i++) { std::string key = EncodeKey(i + 1); - s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]); + s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); ASSERT_OK(s); ASSERT_NE(nullptr, handles[i]); } - ASSERT_EQ(10, cache->GetUsage()); + ASSERT_EQ(10, cache.get()->GetUsage()); // test2: set the flag to true. Insert and check if it fails. std::string extra_key = EncodeKey(100); Value* extra_value = new Value(0); - cache->SetStrictCapacityLimit(true); - Cache::Handle* handle; - s = cache->Insert(extra_key, extra_value, 1, &deleter, &handle); + cache.get()->SetStrictCapacityLimit(true); + TypedHandle* handle; + s = cache.Insert(extra_key, extra_value, 1, &handle); ASSERT_TRUE(s.IsMemoryLimit()); ASSERT_EQ(nullptr, handle); - ASSERT_EQ(10, cache->GetUsage()); + ASSERT_EQ(10, cache.get()->GetUsage()); for (int i = 0; i < 10; i++) { - cache->Release(handles[i]); + cache.Release(handles[i]); } // test3: init with flag being true. - std::shared_ptr cache2 = NewCache(5, 0, true); + SharedCache cache2{NewCache(5, 0, true)}; for (int i = 0; i < 5; i++) { std::string key = EncodeKey(i + 1); - s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]); + s = cache2.Insert(key, new Value(i + 1), 1, &handles[i]); ASSERT_OK(s); ASSERT_NE(nullptr, handles[i]); } - s = cache2->Insert(extra_key, extra_value, 1, &deleter, &handle); + s = cache2.Insert(extra_key, extra_value, 1, &handle); ASSERT_TRUE(s.IsMemoryLimit()); ASSERT_EQ(nullptr, handle); // test insert without handle - s = cache2->Insert(extra_key, extra_value, 1, &deleter); + s = cache2.Insert(extra_key, extra_value, 1); // AS if the key have been inserted into cache but get evicted immediately. ASSERT_OK(s); - ASSERT_EQ(5, cache2->GetUsage()); - ASSERT_EQ(nullptr, cache2->Lookup(extra_key)); + ASSERT_EQ(5, cache2.get()->GetUsage()); + ASSERT_EQ(nullptr, cache2.Lookup(extra_key)); for (int i = 0; i < 5; i++) { - cache2->Release(handles[i]); + cache2.Release(handles[i]); } } @@ -829,55 +823,54 @@ TEST_P(CacheTest, OverCapacity) { size_t n = 10; // a LRUCache with n entries and one shard only - std::shared_ptr cache = NewCache(n, 0, false); - - std::vector handles(n + 1); + SharedCache cache{NewCache(n, 0, false)}; + std::vector handles(n + 1); // Insert n+1 entries, but not releasing. for (int i = 0; i < static_cast(n + 1); i++) { std::string key = EncodeKey(i + 1); - Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]); + Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); ASSERT_TRUE(s.ok()); } // Guess what's in the cache now? for (int i = 0; i < static_cast(n + 1); i++) { std::string key = EncodeKey(i + 1); - auto h = cache->Lookup(key); + auto h = cache.Lookup(key); ASSERT_TRUE(h != nullptr); - if (h) cache->Release(h); + if (h) cache.Release(h); } // the cache is over capacity since nothing could be evicted - ASSERT_EQ(n + 1U, cache->GetUsage()); + ASSERT_EQ(n + 1U, cache.get()->GetUsage()); for (int i = 0; i < static_cast(n + 1); i++) { - cache->Release(handles[i]); + cache.Release(handles[i]); } if (GetParam() == kHyperClock) { // Make sure eviction is triggered. - ASSERT_OK(cache->Insert(EncodeKey(-1), nullptr, 1, &deleter, &handles[0])); + ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, &handles[0])); // cache is under capacity now since elements were released - ASSERT_GE(n, cache->GetUsage()); + ASSERT_GE(n, cache.get()->GetUsage()); // clean up - cache->Release(handles[0]); + cache.Release(handles[0]); } else { // LRUCache checks for over-capacity in Release. // cache is exactly at capacity now with minimal eviction - ASSERT_EQ(n, cache->GetUsage()); + ASSERT_EQ(n, cache.get()->GetUsage()); // element 0 is evicted and the rest is there // This is consistent with the LRU policy since the element 0 // was released first for (int i = 0; i < static_cast(n + 1); i++) { std::string key = EncodeKey(i + 1); - auto h = cache->Lookup(key); + auto h = cache.Lookup(key); if (h) { ASSERT_NE(static_cast(i), 0U); - cache->Release(h); + cache.Release(h); } else { ASSERT_EQ(static_cast(i), 0U); } @@ -885,40 +878,15 @@ TEST_P(CacheTest, OverCapacity) { } } -namespace { -std::vector> legacy_callback_state; -void legacy_callback(void* value, size_t charge) { - legacy_callback_state.push_back( - {DecodeValue(value), static_cast(charge)}); -} -}; // namespace - -TEST_P(CacheTest, ApplyToAllCacheEntriesTest) { - std::vector> inserted; - legacy_callback_state.clear(); - - for (int i = 0; i < 10; ++i) { - Insert(i, i * 2, i + 1); - inserted.push_back({i * 2, i + 1}); - } - cache_->ApplyToAllCacheEntries(legacy_callback, true); - - std::sort(inserted.begin(), inserted.end()); - std::sort(legacy_callback_state.begin(), legacy_callback_state.end()); - ASSERT_EQ(inserted.size(), legacy_callback_state.size()); - for (int i = 0; i < static_cast(inserted.size()); ++i) { - EXPECT_EQ(inserted[i], legacy_callback_state[i]); - } -} - TEST_P(CacheTest, ApplyToAllEntriesTest) { std::vector callback_state; - const auto callback = [&](const Slice& key, void* value, size_t charge, - Cache::DeleterFn deleter) { + const auto callback = [&](const Slice& key, Cache::ObjectPtr value, + size_t charge, + const Cache::CacheItemHelper* helper) { callback_state.push_back(std::to_string(DecodeKey(key)) + "," + std::to_string(DecodeValue(value)) + "," + std::to_string(charge)); - assert(deleter == &CacheTest::Deleter); + assert(helper == &CacheTest::kHelper); }; std::vector inserted; @@ -957,8 +925,8 @@ TEST_P(CacheTest, ApplyToAllEntriesDuringResize) { // For callback int special_count = 0; - const auto callback = [&](const Slice&, void*, size_t charge, - Cache::DeleterFn) { + const auto callback = [&](const Slice&, Cache::ObjectPtr, size_t charge, + const Cache::CacheItemHelper*) { if (charge == static_cast(kSpecialCharge)) { ++special_count; } @@ -1020,7 +988,7 @@ TEST_P(CacheTest, GetChargeAndDeleter) { Cache::Handle* h1 = cache_->Lookup(EncodeKey(1)); ASSERT_EQ(2, DecodeValue(cache_->Value(h1))); ASSERT_EQ(1, cache_->GetCharge(h1)); - ASSERT_EQ(&CacheTest::Deleter, cache_->GetDeleter(h1)); + ASSERT_EQ(&CacheTest::kHelper, cache_->GetCacheItemHelper(h1)); cache_->Release(h1); } diff --git a/cache/charged_cache.cc b/cache/charged_cache.cc index a9ff969b81..3c32fc9611 100644 --- a/cache/charged_cache.cc +++ b/cache/charged_cache.cc @@ -17,25 +17,10 @@ ChargedCache::ChargedCache(std::shared_ptr cache, CacheReservationManagerImpl>( block_cache))) {} -Status ChargedCache::Insert(const Slice& key, void* value, size_t charge, - DeleterFn deleter, Handle** handle, - Priority priority) { - Status s = cache_->Insert(key, value, charge, deleter, handle, priority); - if (s.ok()) { - // Insert may cause the cache entry eviction if the cache is full. So we - // directly call the reservation manager to update the total memory used - // in the cache. - assert(cache_res_mgr_); - cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage()) - .PermitUncheckedError(); - } - return s; -} - -Status ChargedCache::Insert(const Slice& key, void* value, +Status ChargedCache::Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, size_t charge, Handle** handle, Priority priority) { - Status s = cache_->Insert(key, value, helper, charge, handle, priority); + Status s = cache_->Insert(key, obj, helper, charge, handle, priority); if (s.ok()) { // Insert may cause the cache entry eviction if the cache is full. So we // directly call the reservation manager to update the total memory used @@ -47,22 +32,21 @@ Status ChargedCache::Insert(const Slice& key, void* value, return s; } -Cache::Handle* ChargedCache::Lookup(const Slice& key, Statistics* stats) { - return cache_->Lookup(key, stats); -} - Cache::Handle* ChargedCache::Lookup(const Slice& key, const CacheItemHelper* helper, - const CreateCallback& create_cb, + CreateContext* create_context, Priority priority, bool wait, Statistics* stats) { - auto handle = cache_->Lookup(key, helper, create_cb, priority, wait, stats); + auto handle = + cache_->Lookup(key, helper, create_context, priority, wait, stats); // Lookup may promote the KV pair from the secondary cache to the primary // cache. So we directly call the reservation manager to update the total // memory used in the cache. - assert(cache_res_mgr_); - cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage()) - .PermitUncheckedError(); + if (helper && helper->create_cb) { + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage()) + .PermitUncheckedError(); + } return handle; } diff --git a/cache/charged_cache.h b/cache/charged_cache.h index 1739e40889..4bbb667596 100644 --- a/cache/charged_cache.h +++ b/cache/charged_cache.h @@ -23,16 +23,14 @@ class ChargedCache : public Cache { std::shared_ptr block_cache); ~ChargedCache() override = default; - Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, - Handle** handle, Priority priority) override; - Status Insert(const Slice& key, void* value, const CacheItemHelper* helper, + Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, size_t charge, Handle** handle = nullptr, Priority priority = Priority::LOW) override; - Cache::Handle* Lookup(const Slice& key, Statistics* stats) override; Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper, - const CreateCallback& create_cb, Priority priority, - bool wait, Statistics* stats = nullptr) override; + CreateContext* create_context, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr) override; bool Release(Cache::Handle* handle, bool useful, bool erase_if_last_ref = false) override; @@ -56,7 +54,9 @@ class ChargedCache : public Cache { return cache_->HasStrictCapacityLimit(); } - void* Value(Cache::Handle* handle) override { return cache_->Value(handle); } + ObjectPtr Value(Cache::Handle* handle) override { + return cache_->Value(handle); + } bool IsReady(Cache::Handle* handle) override { return cache_->IsReady(handle); @@ -84,22 +84,17 @@ class ChargedCache : public Cache { return cache_->GetCharge(handle); } - Cache::DeleterFn GetDeleter(Cache::Handle* handle) const override { - return cache_->GetDeleter(handle); + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override { + return cache_->GetCacheItemHelper(handle); } void ApplyToAllEntries( - const std::function& callback, + const std::function& callback, const Cache::ApplyToAllEntriesOptions& opts) override { cache_->ApplyToAllEntries(callback, opts); } - void ApplyToAllCacheEntries(void (*callback)(void* value, size_t charge), - bool thread_safe) override { - cache_->ApplyToAllCacheEntries(callback, thread_safe); - } - std::string GetPrintableOptions() const override { return cache_->GetPrintableOptions(); } diff --git a/cache/clock_cache.cc b/cache/clock_cache.cc index 6c9f18c2fb..9476dba7a8 100644 --- a/cache/clock_cache.cc +++ b/cache/clock_cache.cc @@ -50,12 +50,12 @@ inline uint64_t GetInitialCountdown(Cache::Priority priority) { } } -inline void FreeDataMarkEmpty(ClockHandle& h) { +inline void FreeDataMarkEmpty(ClockHandle& h, MemoryAllocator* allocator) { // NOTE: in theory there's more room for parallelism if we copy the handle // data and delay actions like this until after marking the entry as empty, // but performance tests only show a regression by copying the few words // of data. - h.FreeData(); + h.FreeData(allocator); #ifndef NDEBUG // Mark slot as empty, with assertion @@ -115,24 +115,23 @@ inline bool ClockUpdate(ClockHandle& h) { } // namespace -void ClockHandleBasicData::FreeData() const { - if (deleter) { - UniqueId64x2 unhashed; - (*deleter)( - ClockCacheShard::ReverseHash(hashed_key, &unhashed), - value); +void ClockHandleBasicData::FreeData(MemoryAllocator* allocator) const { + if (helper->del_cb) { + helper->del_cb(value, allocator); } } HyperClockTable::HyperClockTable( size_t capacity, bool /*strict_capacity_limit*/, - CacheMetadataChargePolicy metadata_charge_policy, const Opts& opts) + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, const Opts& opts) : length_bits_(CalcHashBits(capacity, opts.estimated_value_size, metadata_charge_policy)), length_bits_mask_((size_t{1} << length_bits_) - 1), occupancy_limit_(static_cast((uint64_t{1} << length_bits_) * kStrictLoadFactor)), - array_(new HandleImpl[size_t{1} << length_bits_]) { + array_(new HandleImpl[size_t{1} << length_bits_]), + allocator_(allocator) { if (metadata_charge_policy == CacheMetadataChargePolicy::kFullChargeCacheMetadata) { usage_ += size_t{GetTableSize()} * sizeof(HandleImpl); @@ -154,7 +153,7 @@ HyperClockTable::~HyperClockTable() { case ClockHandle::kStateInvisible: // rare but possible case ClockHandle::kStateVisible: assert(GetRefcount(h.meta) == 0); - h.FreeData(); + h.FreeData(allocator_); #ifndef NDEBUG Rollback(h.hashed_key, &h); ReclaimEntryUsage(h.GetTotalCharge()); @@ -415,7 +414,7 @@ Status HyperClockTable::Insert(const ClockHandleBasicData& proto, if (handle == nullptr) { // Don't insert the entry but still return ok, as if the entry // inserted into cache and evicted immediately. - proto.FreeData(); + proto.FreeData(allocator_); return Status::OK(); } else { // Need to track usage of fallback detached insert @@ -556,7 +555,7 @@ Status HyperClockTable::Insert(const ClockHandleBasicData& proto, if (handle == nullptr) { revert_usage_fn(); // As if unrefed entry immdiately evicted - proto.FreeData(); + proto.FreeData(allocator_); return Status::OK(); } } @@ -698,14 +697,14 @@ bool HyperClockTable::Release(HandleImpl* h, bool useful, // Took ownership size_t total_charge = h->GetTotalCharge(); if (UNLIKELY(h->IsDetached())) { - h->FreeData(); + h->FreeData(allocator_); // Delete detached handle delete h; detached_usage_.fetch_sub(total_charge, std::memory_order_relaxed); usage_.fetch_sub(total_charge, std::memory_order_relaxed); } else { Rollback(h->hashed_key, h); - FreeDataMarkEmpty(*h); + FreeDataMarkEmpty(*h, allocator_); ReclaimEntryUsage(total_charge); } return true; @@ -790,7 +789,7 @@ void HyperClockTable::Erase(const UniqueId64x2& hashed_key) { // Took ownership assert(hashed_key == h->hashed_key); size_t total_charge = h->GetTotalCharge(); - FreeDataMarkEmpty(*h); + FreeDataMarkEmpty(*h, allocator_); ReclaimEntryUsage(total_charge); // We already have a copy of hashed_key in this case, so OK to // delay Rollback until after releasing the entry @@ -878,7 +877,7 @@ void HyperClockTable::EraseUnRefEntries() { // Took ownership size_t total_charge = h.GetTotalCharge(); Rollback(h.hashed_key, &h); - FreeDataMarkEmpty(h); + FreeDataMarkEmpty(h, allocator_); ReclaimEntryUsage(total_charge); } } @@ -968,7 +967,7 @@ inline void HyperClockTable::Evict(size_t requested_charge, Rollback(h.hashed_key, &h); *freed_charge += h.GetTotalCharge(); *freed_count += 1; - FreeDataMarkEmpty(h); + FreeDataMarkEmpty(h, allocator_); } } @@ -990,9 +989,10 @@ template ClockCacheShard::ClockCacheShard( size_t capacity, bool strict_capacity_limit, CacheMetadataChargePolicy metadata_charge_policy, - const typename Table::Opts& opts) + MemoryAllocator* allocator, const typename Table::Opts& opts) : CacheShardBase(metadata_charge_policy), - table_(capacity, strict_capacity_limit, metadata_charge_policy, opts), + table_(capacity, strict_capacity_limit, metadata_charge_policy, allocator, + opts), capacity_(capacity), strict_capacity_limit_(strict_capacity_limit) { // Initial charge metadata should not exceed capacity @@ -1006,8 +1006,9 @@ void ClockCacheShard
::EraseUnRefEntries() { template void ClockCacheShard
::ApplyToSomeEntries( - const std::function& callback, + const std::function& callback, size_t average_entries_per_lock, size_t* state) { // The state is essentially going to be the starting hash, which works // nicely even if we resize between calls because we use upper-most @@ -1034,7 +1035,7 @@ void ClockCacheShard
::ApplyToSomeEntries( [callback](const HandleImpl& h) { UniqueId64x2 unhashed; callback(ReverseHash(h.hashed_key, &unhashed), h.value, - h.GetTotalCharge(), h.deleter); + h.GetTotalCharge(), h.helper); }, index_begin, index_end, false); } @@ -1078,9 +1079,9 @@ void ClockCacheShard
::SetStrictCapacityLimit( template Status ClockCacheShard
::Insert(const Slice& key, const UniqueId64x2& hashed_key, - void* value, size_t charge, - Cache::DeleterFn deleter, - HandleImpl** handle, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + size_t charge, HandleImpl** handle, Cache::Priority priority) { if (UNLIKELY(key.size() != kCacheKeySize)) { return Status::NotSupported("ClockCache only supports key size " + @@ -1089,7 +1090,7 @@ Status ClockCacheShard
::Insert(const Slice& key, ClockHandleBasicData proto; proto.hashed_key = hashed_key; proto.value = value; - proto.deleter = deleter; + proto.helper = helper; proto.total_charge = charge; Status s = table_.Insert( proto, handle, priority, capacity_.load(std::memory_order_relaxed), @@ -1223,15 +1224,16 @@ HyperClockCache::HyperClockCache( // TODO: should not need to go through two levels of pointer indirection to // get to table entries size_t per_shard = GetPerShardCapacity(); + MemoryAllocator* alloc = this->memory_allocator(); InitShards([=](Shard* cs) { HyperClockTable::Opts opts; opts.estimated_value_size = estimated_value_size; - new (cs) - Shard(per_shard, strict_capacity_limit, metadata_charge_policy, opts); + new (cs) Shard(per_shard, strict_capacity_limit, metadata_charge_policy, + alloc, opts); }); } -void* HyperClockCache::Value(Handle* handle) { +Cache::ObjectPtr HyperClockCache::Value(Handle* handle) { return reinterpret_cast(handle)->value; } @@ -1239,9 +1241,10 @@ size_t HyperClockCache::GetCharge(Handle* handle) const { return reinterpret_cast(handle)->GetTotalCharge(); } -Cache::DeleterFn HyperClockCache::GetDeleter(Handle* handle) const { +const Cache::CacheItemHelper* HyperClockCache::GetCacheItemHelper( + Handle* handle) const { auto h = reinterpret_cast(handle); - return h->deleter; + return h->helper; } namespace { diff --git a/cache/clock_cache.h b/cache/clock_cache.h index ef1b0ccb75..01185849b6 100644 --- a/cache/clock_cache.h +++ b/cache/clock_cache.h @@ -305,8 +305,8 @@ constexpr double kLoadFactor = 0.7; constexpr double kStrictLoadFactor = 0.84; struct ClockHandleBasicData { - void* value = nullptr; - Cache::DeleterFn deleter = nullptr; + Cache::ObjectPtr value = nullptr; + const Cache::CacheItemHelper* helper = nullptr; // A lossless, reversible hash of the fixed-size (16 byte) cache key. This // eliminates the need to store a hash separately. UniqueId64x2 hashed_key = kNullUniqueId64x2; @@ -321,7 +321,7 @@ struct ClockHandleBasicData { inline size_t GetTotalCharge() const { return total_charge; } // Calls deleter (if non-null) on cache key and value - void FreeData() const; + void FreeData(MemoryAllocator* allocator) const; // Required by concept HandleImpl const UniqueId64x2& GetHash() const { return hashed_key; } @@ -411,7 +411,7 @@ class HyperClockTable { HyperClockTable(size_t capacity, bool strict_capacity_limit, CacheMetadataChargePolicy metadata_charge_policy, - const Opts& opts); + MemoryAllocator* allocator, const Opts& opts); ~HyperClockTable(); Status Insert(const ClockHandleBasicData& proto, HandleImpl** handle, @@ -519,6 +519,8 @@ class HyperClockTable { // Updates `detached_usage_` but not `usage_` nor `occupancy_`. inline HandleImpl* DetachedInsert(const ClockHandleBasicData& proto); + MemoryAllocator* GetAllocator() const { return allocator_; } + // Returns the number of bits used to hash an element in the hash // table. static int CalcHashBits(size_t capacity, size_t estimated_value_size, @@ -538,6 +540,9 @@ class HyperClockTable { // Array of slots comprising the hash table. const std::unique_ptr array_; + // From Cache, for deleter + MemoryAllocator* const allocator_; + // We partition the following members into different cache lines // to avoid false sharing among Lookup, Release, Erase and Insert // operations in ClockCacheShard. @@ -563,7 +568,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { public: ClockCacheShard(size_t capacity, bool strict_capacity_limit, CacheMetadataChargePolicy metadata_charge_policy, - const typename Table::Opts& opts); + MemoryAllocator* allocator, const typename Table::Opts& opts); // For CacheShard concept using HandleImpl = typename Table::HandleImpl; @@ -600,9 +605,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { void SetStrictCapacityLimit(bool strict_capacity_limit); - Status Insert(const Slice& key, const UniqueId64x2& hashed_key, void* value, - size_t charge, Cache::DeleterFn deleter, HandleImpl** handle, - Cache::Priority priority); + Status Insert(const Slice& key, const UniqueId64x2& hashed_key, + Cache::ObjectPtr value, const Cache::CacheItemHelper* helper, + size_t charge, HandleImpl** handle, Cache::Priority priority); HandleImpl* Lookup(const Slice& key, const UniqueId64x2& hashed_key); @@ -629,25 +634,18 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { size_t GetTableAddressCount() const; void ApplyToSomeEntries( - const std::function& callback, + const std::function& callback, size_t average_entries_per_lock, size_t* state); void EraseUnRefEntries(); std::string GetPrintableOptions() const { return std::string{}; } - // SecondaryCache not yet supported - Status Insert(const Slice& key, const UniqueId64x2& hashed_key, void* value, - const Cache::CacheItemHelper* helper, size_t charge, - HandleImpl** handle, Cache::Priority priority) { - return Insert(key, hashed_key, value, charge, helper->del_cb, handle, - priority); - } - HandleImpl* Lookup(const Slice& key, const UniqueId64x2& hashed_key, const Cache::CacheItemHelper* /*helper*/, - const Cache::CreateCallback& /*create_cb*/, + Cache::CreateContext* /*create_context*/, Cache::Priority /*priority*/, bool /*wait*/, Statistics* /*stats*/) { return Lookup(key, hashed_key); @@ -686,11 +684,11 @@ class HyperClockCache const char* Name() const override { return "HyperClockCache"; } - void* Value(Handle* handle) override; + Cache::ObjectPtr Value(Handle* handle) override; size_t GetCharge(Handle* handle) const override; - DeleterFn GetDeleter(Handle* handle) const override; + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override; void ReportProblems( const std::shared_ptr& /*info_log*/) const override; diff --git a/cache/compressed_secondary_cache.cc b/cache/compressed_secondary_cache.cc index 7d1bdc7895..23154d4f2a 100644 --- a/cache/compressed_secondary_cache.cc +++ b/cache/compressed_secondary_cache.cc @@ -37,8 +37,10 @@ CompressedSecondaryCache::CompressedSecondaryCache( CompressedSecondaryCache::~CompressedSecondaryCache() { cache_.reset(); } std::unique_ptr CompressedSecondaryCache::Lookup( - const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/, - bool advise_erase, bool& is_in_sec_cache) { + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, + bool& is_in_sec_cache) { + assert(helper); std::unique_ptr handle; is_in_sec_cache = false; Cache::Handle* lru_handle = cache_->Lookup(key); @@ -64,12 +66,14 @@ std::unique_ptr CompressedSecondaryCache::Lookup( ptr = reinterpret_cast(handle_value); handle_value_charge = cache_->GetCharge(lru_handle); } + MemoryAllocator* allocator = cache_options_.memory_allocator.get(); Status s; - void* value{nullptr}; + Cache::ObjectPtr value{nullptr}; size_t charge{0}; if (cache_options_.compression_type == kNoCompression) { - s = create_cb(ptr->get(), handle_value_charge, &value, &charge); + s = helper->create_cb(Slice(ptr->get(), handle_value_charge), + create_context, allocator, &value, &charge); } else { UncompressionContext uncompression_context(cache_options_.compression_type); UncompressionInfo uncompression_info(uncompression_context, @@ -79,14 +83,14 @@ std::unique_ptr CompressedSecondaryCache::Lookup( size_t uncompressed_size{0}; CacheAllocationPtr uncompressed = UncompressData( uncompression_info, (char*)ptr->get(), handle_value_charge, - &uncompressed_size, cache_options_.compress_format_version, - cache_options_.memory_allocator.get()); + &uncompressed_size, cache_options_.compress_format_version, allocator); if (!uncompressed) { cache_->Release(lru_handle, /*erase_if_last_ref=*/true); return nullptr; } - s = create_cb(uncompressed.get(), uncompressed_size, &value, &charge); + s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size), + create_context, allocator, &value, &charge); } if (!s.ok()) { @@ -98,8 +102,9 @@ std::unique_ptr CompressedSecondaryCache::Lookup( cache_->Release(lru_handle, /*erase_if_last_ref=*/true); // Insert a dummy handle. cache_ - ->Insert(key, /*value=*/nullptr, /*charge=*/0, - GetDeletionCallback(cache_options_.enable_custom_split_merge)) + ->Insert(key, /*obj=*/nullptr, + GetHelper(cache_options_.enable_custom_split_merge), + /*charge=*/0) .PermitUncheckedError(); } else { is_in_sec_cache = true; @@ -109,19 +114,20 @@ std::unique_ptr CompressedSecondaryCache::Lookup( return handle; } -Status CompressedSecondaryCache::Insert(const Slice& key, void* value, +Status CompressedSecondaryCache::Insert(const Slice& key, + Cache::ObjectPtr value, const Cache::CacheItemHelper* helper) { if (value == nullptr) { return Status::InvalidArgument(); } Cache::Handle* lru_handle = cache_->Lookup(key); - Cache::DeleterFn del_cb = - GetDeletionCallback(cache_options_.enable_custom_split_merge); + auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge); if (lru_handle == nullptr) { PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1); // Insert a dummy handle if the handle is evicted for the first time. - return cache_->Insert(key, /*value=*/nullptr, /*charge=*/0, del_cb); + return cache_->Insert(key, /*obj=*/nullptr, internal_helper, + /*charge=*/0); } else { cache_->Release(lru_handle, /*erase_if_last_ref=*/false); } @@ -169,10 +175,10 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value, size_t charge{0}; CacheValueChunk* value_chunks_head = SplitValueIntoChunks(val, cache_options_.compression_type, charge); - return cache_->Insert(key, value_chunks_head, charge, del_cb); + return cache_->Insert(key, value_chunks_head, internal_helper, charge); } else { CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr)); - return cache_->Insert(key, buf, size, del_cb); + return cache_->Insert(key, buf, internal_helper, size); } } @@ -276,23 +282,29 @@ CacheAllocationPtr CompressedSecondaryCache::MergeChunksIntoValue( return ptr; } -Cache::DeleterFn CompressedSecondaryCache::GetDeletionCallback( - bool enable_custom_split_merge) { +const Cache::CacheItemHelper* CompressedSecondaryCache::GetHelper( + bool enable_custom_split_merge) const { if (enable_custom_split_merge) { - return [](const Slice& /*key*/, void* obj) { - CacheValueChunk* chunks_head = reinterpret_cast(obj); - while (chunks_head != nullptr) { - CacheValueChunk* tmp_chunk = chunks_head; - chunks_head = chunks_head->next; - tmp_chunk->Free(); - obj = nullptr; - }; - }; + static const Cache::CacheItemHelper kHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) { + CacheValueChunk* chunks_head = static_cast(obj); + while (chunks_head != nullptr) { + CacheValueChunk* tmp_chunk = chunks_head; + chunks_head = chunks_head->next; + tmp_chunk->Free(); + obj = nullptr; + }; + }}; + return &kHelper; } else { - return [](const Slice& /*key*/, void* obj) { - delete reinterpret_cast(obj); - obj = nullptr; - }; + static const Cache::CacheItemHelper kHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) { + delete static_cast(obj); + obj = nullptr; + }}; + return &kHelper; } } diff --git a/cache/compressed_secondary_cache.h b/cache/compressed_secondary_cache.h index 4dee388021..e38a1a861e 100644 --- a/cache/compressed_secondary_cache.h +++ b/cache/compressed_secondary_cache.h @@ -21,7 +21,7 @@ namespace ROCKSDB_NAMESPACE { class CompressedSecondaryCacheResultHandle : public SecondaryCacheResultHandle { public: - CompressedSecondaryCacheResultHandle(void* value, size_t size) + CompressedSecondaryCacheResultHandle(Cache::ObjectPtr value, size_t size) : value_(value), size_(size) {} ~CompressedSecondaryCacheResultHandle() override = default; @@ -34,12 +34,12 @@ class CompressedSecondaryCacheResultHandle : public SecondaryCacheResultHandle { void Wait() override {} - void* Value() override { return value_; } + Cache::ObjectPtr Value() override { return value_; } size_t Size() override { return size_; } private: - void* value_; + Cache::ObjectPtr value_; size_t size_; }; @@ -83,12 +83,13 @@ class CompressedSecondaryCache : public SecondaryCache { const char* Name() const override { return "CompressedSecondaryCache"; } - Status Insert(const Slice& key, void* value, + Status Insert(const Slice& key, Cache::ObjectPtr value, const Cache::CacheItemHelper* helper) override; std::unique_ptr Lookup( - const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/, - bool advise_erase, bool& is_in_sec_cache) override; + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, + bool& is_in_sec_cache) override; bool SupportForceErase() const override { return true; } @@ -129,8 +130,8 @@ class CompressedSecondaryCache : public SecondaryCache { CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head, size_t& charge); - // An implementation of Cache::DeleterFn. - static Cache::DeleterFn GetDeletionCallback(bool enable_custom_split_merge); + // TODO: clean up to use cleaner interfaces in typed_cache.h + const Cache::CacheItemHelper* GetHelper(bool enable_custom_split_merge) const; std::shared_ptr cache_; CompressedSecondaryCacheOptions cache_options_; mutable port::Mutex capacity_mutex_; diff --git a/cache/compressed_secondary_cache_test.cc b/cache/compressed_secondary_cache_test.cc index 574c257a7d..c13b8b390f 100644 --- a/cache/compressed_secondary_cache_test.cc +++ b/cache/compressed_secondary_cache_test.cc @@ -16,7 +16,8 @@ namespace ROCKSDB_NAMESPACE { -class CompressedSecondaryCacheTest : public testing::Test { +class CompressedSecondaryCacheTest : public testing::Test, + public Cache::CreateContext { public: CompressedSecondaryCacheTest() : fail_create_(false) {} ~CompressedSecondaryCacheTest() override = default; @@ -37,13 +38,13 @@ class CompressedSecondaryCacheTest : public testing::Test { size_t size_; }; - static size_t SizeCallback(void* obj) { - return reinterpret_cast(obj)->Size(); + static size_t SizeCallback(Cache::ObjectPtr obj) { + return static_cast(obj)->Size(); } - static Status SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out) { - auto item = reinterpret_cast(from_obj); + static Status SaveToCallback(Cache::ObjectPtr from_obj, size_t from_offset, + size_t length, char* out) { + auto item = static_cast(from_obj); const char* buf = item->Buf(); EXPECT_EQ(length, item->Size()); EXPECT_EQ(from_offset, 0); @@ -51,30 +52,36 @@ class CompressedSecondaryCacheTest : public testing::Test { return Status::OK(); } - static void DeletionCallback(const Slice& /*key*/, void* obj) { - delete reinterpret_cast(obj); + static void DeletionCallback(Cache::ObjectPtr obj, + MemoryAllocator* /*alloc*/) { + delete static_cast(obj); obj = nullptr; } - static Cache::CacheItemHelper helper_; - - static Status SaveToCallbackFail(void* /*obj*/, size_t /*offset*/, - size_t /*size*/, void* /*out*/) { + static Status SaveToCallbackFail(Cache::ObjectPtr /*obj*/, size_t /*offset*/, + size_t /*size*/, char* /*out*/) { return Status::NotSupported(); } - static Cache::CacheItemHelper helper_fail_; - - Cache::CreateCallback test_item_creator = [&](const void* buf, size_t size, - void** out_obj, - size_t* charge) -> Status { - if (fail_create_) { + static Status CreateCallback(const Slice& data, Cache::CreateContext* context, + MemoryAllocator* /*allocator*/, + Cache::ObjectPtr* out_obj, size_t* out_charge) { + auto t = static_cast(context); + if (t->fail_create_) { return Status::NotSupported(); } - *out_obj = reinterpret_cast(new TestItem((char*)buf, size)); - *charge = size; + *out_obj = new TestItem(data.data(), data.size()); + *out_charge = data.size(); return Status::OK(); - }; + } + + static constexpr Cache::CacheItemHelper kHelper{ + CacheEntryRole::kMisc, &DeletionCallback, &SizeCallback, &SaveToCallback, + &CreateCallback}; + + static constexpr Cache::CacheItemHelper kHelperFail{ + CacheEntryRole::kMisc, &DeletionCallback, &SizeCallback, + &SaveToCallbackFail, &CreateCallback}; void SetFailCreate(bool fail) { fail_create_ = fail; } @@ -84,7 +91,7 @@ class CompressedSecondaryCacheTest : public testing::Test { bool is_in_sec_cache{true}; // Lookup an non-existent key. std::unique_ptr handle0 = sec_cache->Lookup( - "k0", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache); + "k0", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache); ASSERT_EQ(handle0, nullptr); Random rnd(301); @@ -92,23 +99,21 @@ class CompressedSecondaryCacheTest : public testing::Test { std::string str1(rnd.RandomString(1000)); TestItem item1(str1.data(), str1.length()); // A dummy handle is inserted if the item is inserted for the first time. - ASSERT_OK(sec_cache->Insert("k1", &item1, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1); ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); std::unique_ptr handle1_1 = sec_cache->Lookup( - "k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache); + "k1", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache); ASSERT_EQ(handle1_1, nullptr); // Insert and Lookup the item k1 for the second time and advise erasing it. - ASSERT_OK(sec_cache->Insert("k1", &item1, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1); std::unique_ptr handle1_2 = sec_cache->Lookup( - "k1", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache); + "k1", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache); ASSERT_NE(handle1_2, nullptr); ASSERT_FALSE(is_in_sec_cache); if (sec_cache_is_compressed) { @@ -128,21 +133,19 @@ class CompressedSecondaryCacheTest : public testing::Test { // Lookup the item k1 again. std::unique_ptr handle1_3 = sec_cache->Lookup( - "k1", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache); + "k1", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache); ASSERT_EQ(handle1_3, nullptr); // Insert and Lookup the item k2. std::string str2(rnd.RandomString(1000)); TestItem item2(str2.data(), str2.length()); - ASSERT_OK(sec_cache->Insert("k2", &item2, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2); std::unique_ptr handle2_1 = sec_cache->Lookup( - "k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache); + "k2", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache); ASSERT_EQ(handle2_1, nullptr); - ASSERT_OK(sec_cache->Insert("k2", &item2, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2); if (sec_cache_is_compressed) { ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, @@ -154,7 +157,7 @@ class CompressedSecondaryCacheTest : public testing::Test { ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); } std::unique_ptr handle2_2 = sec_cache->Lookup( - "k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache); + "k2", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache); ASSERT_NE(handle2_2, nullptr); std::unique_ptr val2 = std::unique_ptr(static_cast(handle2_2->Value())); @@ -223,28 +226,24 @@ class CompressedSecondaryCacheTest : public testing::Test { std::string str1(rnd.RandomString(1000)); TestItem item1(str1.data(), str1.length()); // Insert a dummy handle. - ASSERT_OK(sec_cache->Insert("k1", &item1, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper)); // Insert k1. - ASSERT_OK(sec_cache->Insert("k1", &item1, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper)); // Insert and Lookup the second item. std::string str2(rnd.RandomString(200)); TestItem item2(str2.data(), str2.length()); // Insert a dummy handle, k1 is not evicted. - ASSERT_OK(sec_cache->Insert("k2", &item2, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper)); bool is_in_sec_cache{false}; std::unique_ptr handle1 = sec_cache->Lookup( - "k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache); + "k1", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache); ASSERT_EQ(handle1, nullptr); // Insert k2 and k1 is evicted. - ASSERT_OK(sec_cache->Insert("k2", &item2, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper)); std::unique_ptr handle2 = sec_cache->Lookup( - "k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache); + "k2", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache); ASSERT_NE(handle2, nullptr); std::unique_ptr val2 = std::unique_ptr(static_cast(handle2->Value())); @@ -252,27 +251,24 @@ class CompressedSecondaryCacheTest : public testing::Test { ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0); // Insert k1 again and a dummy handle is inserted. - ASSERT_OK(sec_cache->Insert("k1", &item1, - &CompressedSecondaryCacheTest::helper_)); + ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper)); std::unique_ptr handle1_1 = sec_cache->Lookup( - "k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache); + "k1", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache); ASSERT_EQ(handle1_1, nullptr); // Create Fails. SetFailCreate(true); std::unique_ptr handle2_1 = sec_cache->Lookup( - "k2", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache); + "k2", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache); ASSERT_EQ(handle2_1, nullptr); // Save Fails. std::string str3 = rnd.RandomString(10); TestItem item3(str3.data(), str3.length()); // The Status is OK because a dummy handle is inserted. - ASSERT_OK(sec_cache->Insert("k3", &item3, - &CompressedSecondaryCacheTest::helper_fail_)); - ASSERT_NOK(sec_cache->Insert("k3", &item3, - &CompressedSecondaryCacheTest::helper_fail_)); + ASSERT_OK(sec_cache->Insert("k3", &item3, &kHelperFail)); + ASSERT_NOK(sec_cache->Insert("k3", &item3, &kHelperFail)); sec_cache.reset(); } @@ -309,15 +305,13 @@ class CompressedSecondaryCacheTest : public testing::Test { Random rnd(301); std::string str1 = rnd.RandomString(1001); auto item1_1 = new TestItem(str1.data(), str1.length()); - ASSERT_OK(cache->Insert( - "k1", item1_1, &CompressedSecondaryCacheTest::helper_, str1.length())); + ASSERT_OK(cache->Insert("k1", item1_1, &kHelper, str1.length())); std::string str2 = rnd.RandomString(1012); auto item2_1 = new TestItem(str2.data(), str2.length()); // After this Insert, primary cache contains k2 and secondary cache contains // k1's dummy item. - ASSERT_OK(cache->Insert( - "k2", item2_1, &CompressedSecondaryCacheTest::helper_, str2.length())); + ASSERT_OK(cache->Insert("k2", item2_1, &kHelper, str2.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1); ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); @@ -326,22 +320,19 @@ class CompressedSecondaryCacheTest : public testing::Test { auto item3_1 = new TestItem(str3.data(), str3.length()); // After this Insert, primary cache contains k3 and secondary cache contains // k1's dummy item and k2's dummy item. - ASSERT_OK(cache->Insert( - "k3", item3_1, &CompressedSecondaryCacheTest::helper_, str3.length())); + ASSERT_OK(cache->Insert("k3", item3_1, &kHelper, str3.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2); // After this Insert, primary cache contains k1 and secondary cache contains // k1's dummy item, k2's dummy item, and k3's dummy item. auto item1_2 = new TestItem(str1.data(), str1.length()); - ASSERT_OK(cache->Insert( - "k1", item1_2, &CompressedSecondaryCacheTest::helper_, str1.length())); + ASSERT_OK(cache->Insert("k1", item1_2, &kHelper, str1.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3); // After this Insert, primary cache contains k2 and secondary cache contains // k1's item, k2's dummy item, and k3's dummy item. auto item2_2 = new TestItem(str2.data(), str2.length()); - ASSERT_OK(cache->Insert( - "k2", item2_2, &CompressedSecondaryCacheTest::helper_, str2.length())); + ASSERT_OK(cache->Insert("k2", item2_2, &kHelper, str2.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1); if (sec_cache_is_compressed) { ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, @@ -356,8 +347,7 @@ class CompressedSecondaryCacheTest : public testing::Test { // After this Insert, primary cache contains k3 and secondary cache contains // k1's item and k2's item. auto item3_2 = new TestItem(str3.data(), str3.length()); - ASSERT_OK(cache->Insert( - "k3", item3_2, &CompressedSecondaryCacheTest::helper_, str3.length())); + ASSERT_OK(cache->Insert("k3", item3_2, &kHelper, str3.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2); if (sec_cache_is_compressed) { ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, @@ -370,8 +360,7 @@ class CompressedSecondaryCacheTest : public testing::Test { } Cache::Handle* handle; - handle = cache->Lookup("k3", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k3", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); auto val3 = static_cast(cache->Value(handle)); @@ -380,15 +369,13 @@ class CompressedSecondaryCacheTest : public testing::Test { cache->Release(handle); // Lookup an non-existent key. - handle = cache->Lookup("k0", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k0", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_EQ(handle, nullptr); // This Lookup should just insert a dummy handle in the primary cache // and the k1 is still in the secondary cache. - handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1); @@ -400,8 +387,7 @@ class CompressedSecondaryCacheTest : public testing::Test { // This Lookup should erase k1 from the secondary cache and insert // it into primary cache; then k3 is demoted. // k2 and k3 are in secondary cache. - handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1); @@ -409,8 +395,7 @@ class CompressedSecondaryCacheTest : public testing::Test { cache->Release(handle); // k2 is still in secondary cache. - handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 2); @@ -418,8 +403,7 @@ class CompressedSecondaryCacheTest : public testing::Test { // Testing SetCapacity(). ASSERT_OK(secondary_cache->SetCapacity(0)); - handle = cache->Lookup("k3", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k3", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_EQ(handle, nullptr); @@ -429,35 +413,30 @@ class CompressedSecondaryCacheTest : public testing::Test { ASSERT_EQ(capacity, 7000); auto item1_3 = new TestItem(str1.data(), str1.length()); // After this Insert, primary cache contains k1. - ASSERT_OK(cache->Insert( - "k1", item1_3, &CompressedSecondaryCacheTest::helper_, str2.length())); + ASSERT_OK(cache->Insert("k1", item1_3, &kHelper, str2.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 4); auto item2_3 = new TestItem(str2.data(), str2.length()); // After this Insert, primary cache contains k2 and secondary cache contains // k1's dummy item. - ASSERT_OK(cache->Insert( - "k2", item2_3, &CompressedSecondaryCacheTest::helper_, str1.length())); + ASSERT_OK(cache->Insert("k2", item2_3, &kHelper, str1.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 4); auto item1_4 = new TestItem(str1.data(), str1.length()); // After this Insert, primary cache contains k1 and secondary cache contains // k1's dummy item and k2's dummy item. - ASSERT_OK(cache->Insert( - "k1", item1_4, &CompressedSecondaryCacheTest::helper_, str2.length())); + ASSERT_OK(cache->Insert("k1", item1_4, &kHelper, str2.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 5); auto item2_4 = new TestItem(str2.data(), str2.length()); // After this Insert, primary cache contains k2 and secondary cache contains // k1's real item and k2's dummy item. - ASSERT_OK(cache->Insert( - "k2", item2_4, &CompressedSecondaryCacheTest::helper_, str2.length())); + ASSERT_OK(cache->Insert("k2", item2_4, &kHelper, str2.length())); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 5); // This Lookup should just insert a dummy handle in the primary cache // and the k1 is still in the secondary cache. - handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, + handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); @@ -496,18 +475,13 @@ class CompressedSecondaryCacheTest : public testing::Test { Random rnd(301); std::string str1 = rnd.RandomString(1001); auto item1 = std::make_unique(str1.data(), str1.length()); - ASSERT_NOK(cache->Insert("k1", item1.get(), nullptr, str1.length())); - ASSERT_OK(cache->Insert("k1", item1.get(), - &CompressedSecondaryCacheTest::helper_, - str1.length())); + ASSERT_OK(cache->Insert("k1", item1.get(), &kHelper, str1.length())); item1.release(); // Appease clang-analyze "potential memory leak" Cache::Handle* handle; - handle = cache->Lookup("k2", nullptr, test_item_creator, - Cache::Priority::LOW, true); + handle = cache->Lookup("k2", nullptr, this, Cache::Priority::LOW, true); ASSERT_EQ(handle, nullptr); - handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, false); + handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, false); ASSERT_EQ(handle, nullptr); cache.reset(); @@ -543,29 +517,25 @@ class CompressedSecondaryCacheTest : public testing::Test { Random rnd(301); std::string str1 = rnd.RandomString(1001); auto item1 = new TestItem(str1.data(), str1.length()); - ASSERT_OK(cache->Insert("k1", item1, - &CompressedSecondaryCacheTest::helper_fail_, - str1.length())); + ASSERT_OK(cache->Insert("k1", item1, &kHelperFail, str1.length())); std::string str2 = rnd.RandomString(1002); auto item2 = new TestItem(str2.data(), str2.length()); // k1 should be demoted to the secondary cache. - ASSERT_OK(cache->Insert("k2", item2, - &CompressedSecondaryCacheTest::helper_fail_, - str2.length())); + ASSERT_OK(cache->Insert("k2", item2, &kHelperFail, str2.length())); Cache::Handle* handle; - handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_fail_, - test_item_creator, Cache::Priority::LOW, true); + handle = + cache->Lookup("k2", &kHelperFail, this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); // This lookup should fail, since k1 demotion would have failed. - handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_fail_, - test_item_creator, Cache::Priority::LOW, true); + handle = + cache->Lookup("k1", &kHelperFail, this, Cache::Priority::LOW, true); ASSERT_EQ(handle, nullptr); // Since k1 was not promoted, k2 should still be in cache. - handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_fail_, - test_item_creator, Cache::Priority::LOW, true); + handle = + cache->Lookup("k2", &kHelperFail, this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); @@ -602,28 +572,23 @@ class CompressedSecondaryCacheTest : public testing::Test { Random rnd(301); std::string str1 = rnd.RandomString(1001); auto item1 = new TestItem(str1.data(), str1.length()); - ASSERT_OK(cache->Insert("k1", item1, &CompressedSecondaryCacheTest::helper_, - str1.length())); + ASSERT_OK(cache->Insert("k1", item1, &kHelper, str1.length())); std::string str2 = rnd.RandomString(1002); auto item2 = new TestItem(str2.data(), str2.length()); // k1 should be demoted to the secondary cache. - ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_, - str2.length())); + ASSERT_OK(cache->Insert("k2", item2, &kHelper, str2.length())); Cache::Handle* handle; SetFailCreate(true); - handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); // This lookup should fail, since k1 creation would have failed - handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true); ASSERT_EQ(handle, nullptr); // Since k1 didn't get promoted, k2 should still be in cache - handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); @@ -660,32 +625,27 @@ class CompressedSecondaryCacheTest : public testing::Test { Random rnd(301); std::string str1 = rnd.RandomString(1001); auto item1_1 = new TestItem(str1.data(), str1.length()); - ASSERT_OK(cache->Insert( - "k1", item1_1, &CompressedSecondaryCacheTest::helper_, str1.length())); + ASSERT_OK(cache->Insert("k1", item1_1, &kHelper, str1.length())); std::string str2 = rnd.RandomString(1002); std::string str2_clone{str2}; auto item2 = new TestItem(str2.data(), str2.length()); // After this Insert, primary cache contains k2 and secondary cache contains // k1's dummy item. - ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_, - str2.length())); + ASSERT_OK(cache->Insert("k2", item2, &kHelper, str2.length())); // After this Insert, primary cache contains k1 and secondary cache contains // k1's dummy item and k2's dummy item. auto item1_2 = new TestItem(str1.data(), str1.length()); - ASSERT_OK(cache->Insert( - "k1", item1_2, &CompressedSecondaryCacheTest::helper_, str1.length())); + ASSERT_OK(cache->Insert("k1", item1_2, &kHelper, str1.length())); auto item2_2 = new TestItem(str2.data(), str2.length()); // After this Insert, primary cache contains k2 and secondary cache contains // k1's item and k2's dummy item. - ASSERT_OK(cache->Insert( - "k2", item2_2, &CompressedSecondaryCacheTest::helper_, str2.length())); + ASSERT_OK(cache->Insert("k2", item2_2, &kHelper, str2.length())); Cache::Handle* handle2; - handle2 = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + handle2 = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true); ASSERT_NE(handle2, nullptr); cache->Release(handle2); @@ -693,14 +653,12 @@ class CompressedSecondaryCacheTest : public testing::Test { // strict_capacity_limit is true, but the lookup should still succeed. // A k1's dummy item is inserted into primary cache. Cache::Handle* handle1; - handle1 = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + handle1 = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true); ASSERT_NE(handle1, nullptr); cache->Release(handle1); // Since k1 didn't get inserted, k2 should still be in cache - handle2 = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + handle2 = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true); ASSERT_NE(handle2, nullptr); cache->Release(handle2); @@ -741,7 +699,7 @@ class CompressedSecondaryCacheTest : public testing::Test { current_chunk = current_chunk->next; ASSERT_EQ(current_chunk->size, 98); - sec_cache->GetDeletionCallback(true)("dummy", chunks_head); + sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr); } void MergeChunksIntoValueTest() { @@ -822,23 +780,13 @@ class CompressedSecondaryCacheTest : public testing::Test { std::string value_str{value.get(), charge}; ASSERT_EQ(strcmp(value_str.data(), str.data()), 0); - sec_cache->GetDeletionCallback(true)("dummy", chunks_head); + sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr); } private: bool fail_create_; }; -Cache::CacheItemHelper CompressedSecondaryCacheTest::helper_( - CompressedSecondaryCacheTest::SizeCallback, - CompressedSecondaryCacheTest::SaveToCallback, - CompressedSecondaryCacheTest::DeletionCallback); - -Cache::CacheItemHelper CompressedSecondaryCacheTest::helper_fail_( - CompressedSecondaryCacheTest::SizeCallback, - CompressedSecondaryCacheTest::SaveToCallbackFail, - CompressedSecondaryCacheTest::DeletionCallback); - class CompressedSecCacheTestWithCompressAndAllocatorParam : public CompressedSecondaryCacheTest, public ::testing::WithParamInterface> { diff --git a/cache/lru_cache.cc b/cache/lru_cache.cc index c8e4d29bab..95cd320a7b 100644 --- a/cache/lru_cache.cc +++ b/cache/lru_cache.cc @@ -22,20 +22,28 @@ namespace ROCKSDB_NAMESPACE { namespace lru_cache { +namespace { // A distinct pointer value for marking "dummy" cache entries -void* const kDummyValueMarker = const_cast("kDummyValueMarker"); +struct DummyValue { + char val[12] = "kDummyValue"; +}; +DummyValue kDummyValue{}; +} // namespace -LRUHandleTable::LRUHandleTable(int max_upper_hash_bits) +LRUHandleTable::LRUHandleTable(int max_upper_hash_bits, + MemoryAllocator* allocator) : length_bits_(/* historical starting size*/ 4), list_(new LRUHandle* [size_t{1} << length_bits_] {}), elems_(0), - max_length_bits_(max_upper_hash_bits) {} + max_length_bits_(max_upper_hash_bits), + allocator_(allocator) {} LRUHandleTable::~LRUHandleTable() { + auto alloc = allocator_; ApplyToEntriesRange( - [](LRUHandle* h) { + [alloc](LRUHandle* h) { if (!h->HasRefs()) { - h->Free(); + h->Free(alloc); } }, 0, size_t{1} << length_bits_); @@ -118,6 +126,7 @@ LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit, double low_pri_pool_ratio, bool use_adaptive_mutex, CacheMetadataChargePolicy metadata_charge_policy, int max_upper_hash_bits, + MemoryAllocator* allocator, SecondaryCache* secondary_cache) : CacheShardBase(metadata_charge_policy), capacity_(0), @@ -128,7 +137,7 @@ LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit, high_pri_pool_capacity_(0), low_pri_pool_ratio_(low_pri_pool_ratio), low_pri_pool_capacity_(0), - table_(max_upper_hash_bits), + table_(max_upper_hash_bits, allocator), usage_(0), lru_usage_(0), mutex_(use_adaptive_mutex), @@ -159,13 +168,14 @@ void LRUCacheShard::EraseUnRefEntries() { } for (auto entry : last_reference_list) { - entry->Free(); + entry->Free(table_.GetAllocator()); } } void LRUCacheShard::ApplyToSomeEntries( - const std::function& callback, + const std::function& callback, size_t average_entries_per_lock, size_t* state) { // The state is essentially going to be the starting hash, which works // nicely even if we resize between calls because we use upper-most @@ -192,11 +202,8 @@ void LRUCacheShard::ApplyToSomeEntries( table_.ApplyToEntriesRange( [callback, metadata_charge_policy = metadata_charge_policy_](LRUHandle* h) { - DeleterFn deleter = h->IsSecondaryCacheCompatible() - ? h->info_.helper->del_cb - : h->info_.deleter; callback(h->key(), h->value, h->GetCharge(metadata_charge_policy), - deleter); + h->helper); }, index_begin, index_end); } @@ -339,11 +346,11 @@ void LRUCacheShard::TryInsertIntoSecondaryCache( for (auto entry : evicted_handles) { if (secondary_cache_ && entry->IsSecondaryCacheCompatible() && !entry->IsInSecondaryCache()) { - secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper) + secondary_cache_->Insert(entry->key(), entry->value, entry->helper) .PermitUncheckedError(); } // Free the entries here outside of mutex for performance reasons. - entry->Free(); + entry->Free(table_.GetAllocator()); } } @@ -464,7 +471,7 @@ void LRUCacheShard::Promote(LRUHandle* e) { TryInsertIntoSecondaryCache(last_reference_list); if (free_standalone_handle) { e->Unref(); - e->Free(); + e->Free(table_.GetAllocator()); e = nullptr; } else { PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1); @@ -476,9 +483,9 @@ void LRUCacheShard::Promote(LRUHandle* e) { // rare case that one exists Cache::Priority priority = e->IsHighPri() ? Cache::Priority::HIGH : Cache::Priority::LOW; - s = Insert(e->key(), e->hash, kDummyValueMarker, /*charge=*/0, - /*deleter=*/nullptr, /*helper=*/nullptr, /*handle=*/nullptr, - priority); + s = Insert(e->key(), e->hash, &kDummyValue, &kNoopCacheItemHelper, + /*charge=*/0, + /*handle=*/nullptr, priority); } else { e->SetInCache(true); LRUHandle* handle = e; @@ -508,7 +515,7 @@ void LRUCacheShard::Promote(LRUHandle* e) { LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash, const Cache::CacheItemHelper* helper, - const Cache::CreateCallback& create_cb, + Cache::CreateContext* create_context, Cache::Priority priority, bool wait, Statistics* stats) { LRUHandle* e = nullptr; @@ -518,7 +525,7 @@ LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash, e = table_.Lookup(key, hash); if (e != nullptr) { assert(e->InCache()); - if (e->value == kDummyValueMarker) { + if (e->value == &kDummyValue) { // For a dummy handle, if it was retrieved from secondary cache, // it may still exist in secondary cache. // If the handle exists in secondary cache, the value should be @@ -547,24 +554,17 @@ LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash, // standalone handle is returned to the caller. Only if the block is hit // again, we erase it from CompressedSecondaryCache and add it into the // primary cache. - if (!e && secondary_cache_ && helper && helper->saveto_cb) { - // For objects from the secondary cache, we expect the caller to provide - // a way to create/delete the primary cache object. The only case where - // a deleter would not be required is for dummy entries inserted for - // accounting purposes, which we won't demote to the secondary cache - // anyway. - assert(create_cb && helper->del_cb); + if (!e && secondary_cache_ && helper && helper->create_cb) { bool is_in_sec_cache{false}; std::unique_ptr secondary_handle = - secondary_cache_->Lookup(key, create_cb, wait, found_dummy_entry, - is_in_sec_cache); + secondary_cache_->Lookup(key, helper, create_context, wait, + found_dummy_entry, is_in_sec_cache); if (secondary_handle != nullptr) { e = static_cast(malloc(sizeof(LRUHandle) - 1 + key.size())); e->m_flags = 0; e->im_flags = 0; - e->SetSecondaryCacheCompatible(true); - e->info_.helper = helper; + e->helper = helper; e->key_length = key.size(); e->hash = hash; e->refs = 0; @@ -585,7 +585,7 @@ LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash, if (!e->value) { // The secondary cache returned a handle, but the lookup failed. e->Unref(); - e->Free(); + e->Free(table_.GetAllocator()); e = nullptr; } else { PERF_COUNTER_ADD(secondary_cache_hit_count, 1); @@ -669,16 +669,18 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/, // Free the entry here outside of mutex for performance reasons. if (last_reference) { - e->Free(); + e->Free(table_.GetAllocator()); } return last_reference; } -Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value, - size_t charge, - void (*deleter)(const Slice& key, void* value), +Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, + Cache::ObjectPtr value, const Cache::CacheItemHelper* helper, - LRUHandle** handle, Cache::Priority priority) { + size_t charge, LRUHandle** handle, + Cache::Priority priority) { + assert(helper); + // Allocate the memory here outside of the mutex. // If the cache is full, we'll have to release it. // It shouldn't happen very often though. @@ -688,17 +690,7 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value, e->value = value; e->m_flags = 0; e->im_flags = 0; - if (helper) { - // Use only one of the two parameters - assert(deleter == nullptr); - // value == nullptr is reserved for indicating failure for when secondary - // cache compatible - assert(value != nullptr); - e->SetSecondaryCacheCompatible(true); - e->info_.helper = helper; - } else { - e->info_.deleter = deleter; - } + e->helper = helper; e->key_length = key.size(); e->hash = hash; e->refs = 0; @@ -708,6 +700,10 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value, memcpy(e->key_data, key.data(), key.size()); e->CalcTotalCharge(charge, metadata_charge_policy_); + // value == nullptr is reserved for indicating failure for when secondary + // cache compatible + assert(!(e->IsSecondaryCacheCompatible() && value == nullptr)); + return InsertItem(e, handle, /* free_handle_on_fail */ true); } @@ -733,7 +729,7 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) { // Free the entry here outside of mutex for performance reasons. // last_reference will only be true if e != nullptr. if (last_reference) { - e->Free(); + e->Free(table_.GetAllocator()); } } @@ -793,18 +789,19 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits, secondary_cache_(std::move(_secondary_cache)) { size_t per_shard = GetPerShardCapacity(); SecondaryCache* secondary_cache = secondary_cache_.get(); + MemoryAllocator* alloc = memory_allocator(); InitShards([=](LRUCacheShard* cs) { new (cs) LRUCacheShard( per_shard, strict_capacity_limit, high_pri_pool_ratio, low_pri_pool_ratio, use_adaptive_mutex, metadata_charge_policy, - /* max_upper_hash_bits */ 32 - num_shard_bits, secondary_cache); + /* max_upper_hash_bits */ 32 - num_shard_bits, alloc, secondary_cache); }); } -void* LRUCache::Value(Handle* handle) { +Cache::ObjectPtr LRUCache::Value(Handle* handle) { auto h = reinterpret_cast(handle); assert(!h->IsPending() || h->value == nullptr); - assert(h->value != kDummyValueMarker); + assert(h->value != &kDummyValue); return h->value; } @@ -813,13 +810,10 @@ size_t LRUCache::GetCharge(Handle* handle) const { GetShard(0).metadata_charge_policy_); } -Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const { +const Cache::CacheItemHelper* LRUCache::GetCacheItemHelper( + Handle* handle) const { auto h = reinterpret_cast(handle); - if (h->IsSecondaryCacheCompatible()) { - return h->info_.helper->del_cb; - } else { - return h->info_.deleter; - } + return h->helper; } size_t LRUCache::TEST_GetLRUSize() { diff --git a/cache/lru_cache.h b/cache/lru_cache.h index 99b2f2b204..1edccd0ce2 100644 --- a/cache/lru_cache.h +++ b/cache/lru_cache.h @@ -13,6 +13,7 @@ #include "cache/sharded_cache.h" #include "port/lang.h" +#include "port/likely.h" #include "port/malloc.h" #include "port/port.h" #include "rocksdb/secondary_cache.h" @@ -48,13 +49,8 @@ namespace lru_cache { // While refs > 0, public properties like value and deleter must not change. struct LRUHandle { - void* value; - union Info { - Info() {} - ~Info() {} - Cache::DeleterFn deleter; - const Cache::CacheItemHelper* helper; - } info_; + Cache::ObjectPtr value; + const Cache::CacheItemHelper* helper; // An entry is not added to the LRUHandleTable until the secondary cache // lookup is complete, so its safe to have this union. union { @@ -93,14 +89,12 @@ struct LRUHandle { IM_IS_HIGH_PRI = (1 << 0), // Whether this entry is low priority entry. IM_IS_LOW_PRI = (1 << 1), - // Can this be inserted into the secondary cache. - IM_IS_SECONDARY_CACHE_COMPATIBLE = (1 << 2), // Is the handle still being read from a lower tier. - IM_IS_PENDING = (1 << 3), + IM_IS_PENDING = (1 << 2), // Whether this handle is still in a lower tier - IM_IS_IN_SECONDARY_CACHE = (1 << 4), + IM_IS_IN_SECONDARY_CACHE = (1 << 3), // Marks result handles that should not be inserted into cache - IM_IS_STANDALONE = (1 << 5), + IM_IS_STANDALONE = (1 << 4), }; // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!) @@ -130,9 +124,7 @@ struct LRUHandle { bool IsLowPri() const { return im_flags & IM_IS_LOW_PRI; } bool InLowPriPool() const { return m_flags & M_IN_LOW_PRI_POOL; } bool HasHit() const { return m_flags & M_HAS_HIT; } - bool IsSecondaryCacheCompatible() const { - return im_flags & IM_IS_SECONDARY_CACHE_COMPATIBLE; - } + bool IsSecondaryCacheCompatible() const { return helper->size_cb != nullptr; } bool IsPending() const { return im_flags & IM_IS_PENDING; } bool IsInSecondaryCache() const { return im_flags & IM_IS_IN_SECONDARY_CACHE; @@ -178,14 +170,6 @@ struct LRUHandle { void SetHit() { m_flags |= M_HAS_HIT; } - void SetSecondaryCacheCompatible(bool compat) { - if (compat) { - im_flags |= IM_IS_SECONDARY_CACHE_COMPATIBLE; - } else { - im_flags &= ~IM_IS_SECONDARY_CACHE_COMPATIBLE; - } - } - void SetIsPending(bool pending) { if (pending) { im_flags |= IM_IS_PENDING; @@ -210,22 +194,19 @@ struct LRUHandle { } } - void Free() { + void Free(MemoryAllocator* allocator) { assert(refs == 0); - if (!IsSecondaryCacheCompatible() && info_.deleter) { - (*info_.deleter)(key(), value); - } else if (IsSecondaryCacheCompatible()) { - if (IsPending()) { - assert(sec_handle != nullptr); - SecondaryCacheResultHandle* tmp_sec_handle = sec_handle; - tmp_sec_handle->Wait(); - value = tmp_sec_handle->Value(); - delete tmp_sec_handle; - } - if (value) { - (*info_.helper->del_cb)(key(), value); - } + if (UNLIKELY(IsPending())) { + assert(sec_handle != nullptr); + SecondaryCacheResultHandle* tmp_sec_handle = sec_handle; + tmp_sec_handle->Wait(); + value = tmp_sec_handle->Value(); + delete tmp_sec_handle; + } + assert(helper); + if (helper->del_cb) { + helper->del_cb(value, allocator); } free(this); @@ -267,7 +248,7 @@ struct LRUHandle { // 4.4.3's builtin hashtable. class LRUHandleTable { public: - explicit LRUHandleTable(int max_upper_hash_bits); + explicit LRUHandleTable(int max_upper_hash_bits, MemoryAllocator* allocator); ~LRUHandleTable(); LRUHandle* Lookup(const Slice& key, uint32_t hash); @@ -291,6 +272,8 @@ class LRUHandleTable { size_t GetOccupancyCount() const { return elems_; } + MemoryAllocator* GetAllocator() const { return allocator_; } + private: // Return a pointer to slot that points to a cache entry that // matches key/hash. If there is no such cache entry, return a @@ -312,6 +295,9 @@ class LRUHandleTable { // Set from max_upper_hash_bits (see constructor). const int max_length_bits_; + + // From Cache, needed for delete + MemoryAllocator* const allocator_; }; // A single shard of sharded cache. @@ -321,7 +307,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase { double high_pri_pool_ratio, double low_pri_pool_ratio, bool use_adaptive_mutex, CacheMetadataChargePolicy metadata_charge_policy, - int max_upper_hash_bits, SecondaryCache* secondary_cache); + int max_upper_hash_bits, MemoryAllocator* allocator, + SecondaryCache* secondary_cache); public: // Type definitions expected as parameter to ShardedCache using HandleImpl = LRUHandle; @@ -348,26 +335,15 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase { void SetLowPriorityPoolRatio(double low_pri_pool_ratio); // Like Cache methods, but with an extra "hash" parameter. - inline Status Insert(const Slice& key, uint32_t hash, void* value, - size_t charge, Cache::DeleterFn deleter, - LRUHandle** handle, Cache::Priority priority) { - return Insert(key, hash, value, charge, deleter, nullptr, handle, priority); - } - inline Status Insert(const Slice& key, uint32_t hash, void* value, - const Cache::CacheItemHelper* helper, size_t charge, - LRUHandle** handle, Cache::Priority priority) { - assert(helper); - return Insert(key, hash, value, charge, nullptr, helper, handle, priority); - } - // If helper_cb is null, the values of the following arguments don't matter. + Status Insert(const Slice& key, uint32_t hash, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, size_t charge, + LRUHandle** handle, Cache::Priority priority); + LRUHandle* Lookup(const Slice& key, uint32_t hash, const Cache::CacheItemHelper* helper, - const Cache::CreateCallback& create_cb, + Cache::CreateContext* create_context, Cache::Priority priority, bool wait, Statistics* stats); - inline LRUHandle* Lookup(const Slice& key, uint32_t hash) { - return Lookup(key, hash, nullptr, nullptr, Cache::Priority::LOW, true, - nullptr); - } + bool Release(LRUHandle* handle, bool useful, bool erase_if_last_ref); bool IsReady(LRUHandle* /*handle*/); void Wait(LRUHandle* /*handle*/) {} @@ -384,8 +360,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase { size_t GetTableAddressCount() const; void ApplyToSomeEntries( - const std::function& callback, + const std::function& callback, size_t average_entries_per_lock, size_t* state); void EraseUnRefEntries(); @@ -414,9 +391,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase { // nullptr. Status InsertItem(LRUHandle* item, LRUHandle** handle, bool free_handle_on_fail); - Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge, - DeleterFn deleter, const Cache::CacheItemHelper* helper, - LRUHandle** handle, Cache::Priority priority); // Promote an item looked up from the secondary cache to the LRU cache. // The item may be still in the secondary cache. // It is only inserted into the hash table and not the LRU list, and only @@ -521,9 +495,9 @@ class LRUCache kDontChargeCacheMetadata, std::shared_ptr secondary_cache = nullptr); const char* Name() const override { return "LRUCache"; } - void* Value(Handle* handle) override; + ObjectPtr Value(Handle* handle) override; size_t GetCharge(Handle* handle) const override; - DeleterFn GetDeleter(Handle* handle) const override; + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override; void WaitAll(std::vector& handles) override; // Retrieves number of elements in LRU, for unit test purpose only. diff --git a/cache/lru_cache_test.cc b/cache/lru_cache_test.cc index 7904a196d6..f84312cb3f 100644 --- a/cache/lru_cache_test.cc +++ b/cache/lru_cache_test.cc @@ -10,6 +10,7 @@ #include "cache/cache_key.h" #include "cache/clock_cache.h" +#include "cache_helpers.h" #include "db/db_test_util.h" #include "file/sst_file_manager_impl.h" #include "port/port.h" @@ -19,6 +20,7 @@ #include "rocksdb/sst_file_manager.h" #include "rocksdb/utilities/cache_dump_load.h" #include "test_util/testharness.h" +#include "typed_cache.h" #include "util/coding.h" #include "util/random.h" #include "utilities/cache_dump_load_impl.h" @@ -49,14 +51,15 @@ class LRUCacheTest : public testing::Test { high_pri_pool_ratio, low_pri_pool_ratio, use_adaptive_mutex, kDontChargeCacheMetadata, /*max_upper_hash_bits=*/24, + /*allocator*/ nullptr, /*secondary_cache=*/nullptr); } void Insert(const std::string& key, Cache::Priority priority = Cache::Priority::LOW) { - EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/, - nullptr /*deleter*/, nullptr /*handle*/, - priority)); + EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, + &kNoopCacheItemHelper, 1 /*charge*/, + nullptr /*handle*/, priority)); } void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) { @@ -64,7 +67,8 @@ class LRUCacheTest : public testing::Test { } bool Lookup(const std::string& key) { - auto handle = cache_->Lookup(key, 0 /*hash*/); + auto handle = cache_->Lookup(key, 0 /*hash*/, nullptr, nullptr, + Cache::Priority::LOW, true, nullptr); if (handle) { cache_->Release(handle, true /*useful*/, false /*erase*/); return true; @@ -389,15 +393,15 @@ class ClockCacheTest : public testing::Test { Table::Opts opts; opts.estimated_value_size = 1; - new (shard_) - Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata, opts); + new (shard_) Shard(capacity, strict_capacity_limit, + kDontChargeCacheMetadata, /*allocator*/ nullptr, opts); } Status Insert(const UniqueId64x2& hashed_key, Cache::Priority priority = Cache::Priority::LOW) { return shard_->Insert(TestKey(hashed_key), hashed_key, nullptr /*value*/, - 1 /*charge*/, nullptr /*deleter*/, nullptr /*handle*/, - priority); + &kNoopCacheItemHelper, 1 /*charge*/, + nullptr /*handle*/, priority); } Status Insert(char key, Cache::Priority priority = Cache::Priority::LOW) { @@ -407,8 +411,8 @@ class ClockCacheTest : public testing::Test { Status InsertWithLen(char key, size_t len) { std::string skey(len, key); return shard_->Insert(skey, TestHashedKey(key), nullptr /*value*/, - 1 /*charge*/, nullptr /*deleter*/, nullptr /*handle*/, - Cache::Priority::LOW); + &kNoopCacheItemHelper, 1 /*charge*/, + nullptr /*handle*/, Cache::Priority::LOW); } bool Lookup(const Slice& key, const UniqueId64x2& hashed_key, @@ -482,7 +486,7 @@ TEST_F(ClockCacheTest, Limits) { // Single entry charge beyond capacity { Status s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, - 5 /*charge*/, nullptr /*deleter*/, + &kNoopCacheItemHelper, 5 /*charge*/, nullptr /*handle*/, Cache::Priority::LOW); if (strict_capacity_limit) { EXPECT_TRUE(s.IsMemoryLimit()); @@ -495,7 +499,7 @@ TEST_F(ClockCacheTest, Limits) { { HandleImpl* h; ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, - 3 /*charge*/, nullptr /*deleter*/, &h, + &kNoopCacheItemHelper, 3 /*charge*/, &h, Cache::Priority::LOW)); // Try to insert more Status s = Insert('a'); @@ -519,8 +523,9 @@ TEST_F(ClockCacheTest, Limits) { for (size_t i = 0; i < n && s.ok(); ++i) { hkey[1] = i; s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, - (i + kCapacity < n) ? 0 : 1 /*charge*/, - nullptr /*deleter*/, &ha[i], Cache::Priority::LOW); + &kNoopCacheItemHelper, + (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i], + Cache::Priority::LOW); if (i == 0) { EXPECT_OK(s); } @@ -658,18 +663,25 @@ TEST_F(ClockCacheTest, ClockEvictionTest) { } } -void IncrementIntDeleter(const Slice& /*key*/, void* value) { - *reinterpret_cast(value) += 1; -} +namespace { +struct DeleteCounter { + int deleted = 0; +}; +const Cache::CacheItemHelper kDeleteCounterHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + static_cast(value)->deleted += 1; + }}; +} // namespace // Testing calls to CorrectNearOverflow in Release TEST_F(ClockCacheTest, ClockCounterOverflowTest) { NewShard(6, /*strict_capacity_limit*/ false); HandleImpl* h; - int deleted = 0; + DeleteCounter val; UniqueId64x2 hkey = TestHashedKey('x'); - ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &deleted, 1, - IncrementIntDeleter, &h, Cache::Priority::HIGH)); + ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, 1, + &h, Cache::Priority::HIGH)); // Some large number outstanding shard_->TEST_RefN(h, 123456789); @@ -689,18 +701,18 @@ TEST_F(ClockCacheTest, ClockCounterOverflowTest) { // Free all but last 1 shard_->TEST_ReleaseN(h, 123456789); // Still alive - ASSERT_EQ(deleted, 0); + ASSERT_EQ(val.deleted, 0); // Free last ref, which will finalize erasure shard_->Release(h); // Deleted - ASSERT_EQ(deleted, 1); + ASSERT_EQ(val.deleted, 1); } // This test is mostly to exercise some corner case logic, by forcing two // keys to have the same hash, and more TEST_F(ClockCacheTest, CollidingInsertEraseTest) { NewShard(6, /*strict_capacity_limit*/ false); - int deleted = 0; + DeleteCounter val; UniqueId64x2 hkey1 = TestHashedKey('x'); Slice key1 = TestKey(hkey1); UniqueId64x2 hkey2 = TestHashedKey('y'); @@ -708,13 +720,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { UniqueId64x2 hkey3 = TestHashedKey('z'); Slice key3 = TestKey(hkey3); HandleImpl* h1; - ASSERT_OK(shard_->Insert(key1, hkey1, &deleted, 1, IncrementIntDeleter, &h1, + ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1, Cache::Priority::HIGH)); HandleImpl* h2; - ASSERT_OK(shard_->Insert(key2, hkey2, &deleted, 1, IncrementIntDeleter, &h2, + ASSERT_OK(shard_->Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2, Cache::Priority::HIGH)); HandleImpl* h3; - ASSERT_OK(shard_->Insert(key3, hkey3, &deleted, 1, IncrementIntDeleter, &h3, + ASSERT_OK(shard_->Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3, Cache::Priority::HIGH)); // Can repeatedly lookup+release despite the hash collision @@ -739,7 +751,7 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { shard_->Erase(key1, hkey1); // All still alive - ASSERT_EQ(deleted, 0); + ASSERT_EQ(val.deleted, 0); // Invisible to Lookup tmp_h = shard_->Lookup(key1, hkey1); @@ -757,8 +769,8 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { } // Also Insert with invisible entry there - ASSERT_OK(shard_->Insert(key1, hkey1, &deleted, 1, IncrementIntDeleter, - nullptr, Cache::Priority::HIGH)); + ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr, + Cache::Priority::HIGH)); tmp_h = shard_->Lookup(key1, hkey1); // Found but distinct handle ASSERT_NE(nullptr, tmp_h); @@ -766,13 +778,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { ASSERT_TRUE(shard_->Release(tmp_h, /*erase_if_last_ref*/ true)); // tmp_h deleted - ASSERT_EQ(deleted--, 1); + ASSERT_EQ(val.deleted--, 1); // Release last ref on h1 (already invisible) ASSERT_TRUE(shard_->Release(h1, /*erase_if_last_ref*/ false)); // h1 deleted - ASSERT_EQ(deleted--, 1); + ASSERT_EQ(val.deleted--, 1); h1 = nullptr; // Can still find h2, h3 @@ -790,7 +802,7 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { ASSERT_FALSE(shard_->Release(h2, /*erase_if_last_ref*/ false)); // h2 still not deleted (unreferenced in cache) - ASSERT_EQ(deleted, 0); + ASSERT_EQ(val.deleted, 0); // Can still find it tmp_h = shard_->Lookup(key2, hkey2); @@ -800,7 +812,7 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { ASSERT_TRUE(shard_->Release(h2, /*erase_if_last_ref*/ true)); // h2 deleted - ASSERT_EQ(deleted--, 1); + ASSERT_EQ(val.deleted--, 1); tmp_h = shard_->Lookup(key2, hkey2); ASSERT_EQ(nullptr, tmp_h); @@ -815,13 +827,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { ASSERT_FALSE(shard_->Release(h3, /*erase_if_last_ref*/ false)); // h3 still not deleted (unreferenced in cache) - ASSERT_EQ(deleted, 0); + ASSERT_EQ(val.deleted, 0); // Explicit erase shard_->Erase(key3, hkey3); // h3 deleted - ASSERT_EQ(deleted--, 1); + ASSERT_EQ(val.deleted--, 1); tmp_h = shard_->Lookup(key3, hkey3); ASSERT_EQ(nullptr, tmp_h); } @@ -884,12 +896,12 @@ class TestSecondaryCache : public SecondaryCache { using ResultMap = std::unordered_map; explicit TestSecondaryCache(size_t capacity) - : num_inserts_(0), num_lookups_(0), inject_failure_(false) { - cache_ = - NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, nullptr, - kDefaultToAdaptiveMutex, kDontChargeCacheMetadata); - } - ~TestSecondaryCache() override { cache_.reset(); } + : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, + nullptr, kDefaultToAdaptiveMutex, + kDontChargeCacheMetadata)), + num_inserts_(0), + num_lookups_(0), + inject_failure_(false) {} const char* Name() const override { return "TestSecondaryCache"; } @@ -897,7 +909,7 @@ class TestSecondaryCache : public SecondaryCache { void ResetInjectFailure() { inject_failure_ = false; } - Status Insert(const Slice& key, void* value, + Status Insert(const Slice& key, Cache::ObjectPtr value, const Cache::CacheItemHelper* helper) override { if (inject_failure_) { return Status::Corruption("Insertion Data Corrupted"); @@ -916,14 +928,12 @@ class TestSecondaryCache : public SecondaryCache { delete[] buf; return s; } - return cache_->Insert(key, buf, size, - [](const Slice& /*key*/, void* val) -> void { - delete[] static_cast(val); - }); + return cache_.Insert(key, buf, size); } std::unique_ptr Lookup( - const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/, + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, bool /*advise_erase*/, bool& is_in_sec_cache) override { std::string key_str = key.ToString(); TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str); @@ -939,24 +949,25 @@ class TestSecondaryCache : public SecondaryCache { return secondary_handle; } - Cache::Handle* handle = cache_->Lookup(key); + TypedHandle* handle = cache_.Lookup(key); num_lookups_++; if (handle) { - void* value = nullptr; + Cache::ObjectPtr value = nullptr; size_t charge = 0; Status s; if (type != ResultType::DEFER_AND_FAIL) { - char* ptr = (char*)cache_->Value(handle); + char* ptr = cache_.Value(handle); size_t size = DecodeFixed64(ptr); ptr += sizeof(uint64_t); - s = create_cb(ptr, size, &value, &charge); + s = helper->create_cb(Slice(ptr, size), create_context, + /*alloc*/ nullptr, &value, &charge); } if (s.ok()) { secondary_handle.reset(new TestSecondaryCacheResultHandle( cache_.get(), handle, value, charge, type)); is_in_sec_cache = true; } else { - cache_->Release(handle); + cache_.Release(handle); } } return secondary_handle; @@ -995,7 +1006,8 @@ class TestSecondaryCache : public SecondaryCache { class TestSecondaryCacheResultHandle : public SecondaryCacheResultHandle { public: TestSecondaryCacheResultHandle(Cache* cache, Cache::Handle* handle, - void* value, size_t size, ResultType type) + Cache::ObjectPtr value, size_t size, + ResultType type) : cache_(cache), handle_(handle), value_(value), @@ -1012,7 +1024,7 @@ class TestSecondaryCache : public SecondaryCache { void Wait() override {} - void* Value() override { + Cache::ObjectPtr Value() override { assert(is_ready_); return value_; } @@ -1024,12 +1036,15 @@ class TestSecondaryCache : public SecondaryCache { private: Cache* cache_; Cache::Handle* handle_; - void* value_; + Cache::ObjectPtr value_; size_t size_; bool is_ready_; }; - std::shared_ptr cache_; + using SharedCache = + BasicTypedSharedCacheInterface; + using TypedHandle = SharedCache::TypedHandle; + SharedCache cache_; uint32_t num_inserts_; uint32_t num_lookups_; bool inject_failure_; @@ -1049,7 +1064,8 @@ class DBSecondaryCacheTest : public DBTestBase { std::unique_ptr fault_env_; }; -class LRUCacheSecondaryCacheTest : public LRUCacheTest { +class LRUCacheSecondaryCacheTest : public LRUCacheTest, + public Cache::CreateContext { public: LRUCacheSecondaryCacheTest() : fail_create_(false) {} ~LRUCacheSecondaryCacheTest() {} @@ -1071,13 +1087,13 @@ class LRUCacheSecondaryCacheTest : public LRUCacheTest { size_t size_; }; - static size_t SizeCallback(void* obj) { - return reinterpret_cast(obj)->Size(); + static size_t SizeCallback(Cache::ObjectPtr obj) { + return static_cast(obj)->Size(); } - static Status SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out) { - TestItem* item = reinterpret_cast(from_obj); + static Status SaveToCallback(Cache::ObjectPtr from_obj, size_t from_offset, + size_t length, char* out) { + TestItem* item = static_cast(from_obj); char* buf = item->Buf(); EXPECT_EQ(length, item->Size()); EXPECT_EQ(from_offset, 0); @@ -1085,27 +1101,30 @@ class LRUCacheSecondaryCacheTest : public LRUCacheTest { return Status::OK(); } - static void DeletionCallback(const Slice& /*key*/, void* obj) { - delete reinterpret_cast(obj); + static void DeletionCallback(Cache::ObjectPtr obj, + MemoryAllocator* /*alloc*/) { + delete static_cast(obj); } static Cache::CacheItemHelper helper_; - static Status SaveToCallbackFail(void* /*obj*/, size_t /*offset*/, - size_t /*size*/, void* /*out*/) { + static Status SaveToCallbackFail(Cache::ObjectPtr /*from_obj*/, + size_t /*from_offset*/, size_t /*length*/, + char* /*out*/) { return Status::NotSupported(); } static Cache::CacheItemHelper helper_fail_; - Cache::CreateCallback test_item_creator = [&](const void* buf, size_t size, - void** out_obj, - size_t* charge) -> Status { - if (fail_create_) { + static Status CreateCallback(const Slice& data, Cache::CreateContext* context, + MemoryAllocator* /*allocator*/, + Cache::ObjectPtr* out_obj, size_t* out_charge) { + auto t = static_cast(context); + if (t->fail_create_) { return Status::NotSupported(); } - *out_obj = reinterpret_cast(new TestItem((char*)buf, size)); - *charge = size; + *out_obj = new TestItem(data.data(), data.size()); + *out_charge = data.size(); return Status::OK(); }; @@ -1115,15 +1134,17 @@ class LRUCacheSecondaryCacheTest : public LRUCacheTest { bool fail_create_; }; -Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_( +Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_{ + CacheEntryRole::kMisc, LRUCacheSecondaryCacheTest::DeletionCallback, LRUCacheSecondaryCacheTest::SizeCallback, LRUCacheSecondaryCacheTest::SaveToCallback, - LRUCacheSecondaryCacheTest::DeletionCallback); + LRUCacheSecondaryCacheTest::CreateCallback}; -Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_fail_( +Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_fail_{ + CacheEntryRole::kMisc, LRUCacheSecondaryCacheTest::DeletionCallback, LRUCacheSecondaryCacheTest::SizeCallback, LRUCacheSecondaryCacheTest::SaveToCallbackFail, - LRUCacheSecondaryCacheTest::DeletionCallback); + LRUCacheSecondaryCacheTest::CreateCallback}; TEST_F(LRUCacheSecondaryCacheTest, BasicTest) { LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */, @@ -1159,7 +1180,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) { Cache::Handle* handle; handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, stats.get()); + /*context*/ this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str2.size()); cache->Release(handle); @@ -1167,7 +1188,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) { // This lookup should promote k1 and demote k2 handle = cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, stats.get()); + /*context*/ this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str1.size()); cache->Release(handle); @@ -1175,7 +1196,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) { // This lookup should promote k3 and demote k1 handle = cache->Lookup(k3.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true, stats.get()); + /*context*/ this, Cache::Priority::LOW, true, stats.get()); ASSERT_NE(handle, nullptr); ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str3.size()); cache->Release(handle); @@ -1207,18 +1228,19 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicFailTest) { Random rnd(301); std::string str1 = rnd.RandomString(1020); auto item1 = std::make_unique(str1.data(), str1.length()); - ASSERT_TRUE(cache->Insert(k1.AsSlice(), item1.get(), nullptr, str1.length()) - .IsInvalidArgument()); + // NOTE: changed to assert helper != nullptr for efficiency / code size + // ASSERT_TRUE(cache->Insert(k1.AsSlice(), item1.get(), nullptr, + // str1.length()).IsInvalidArgument()); ASSERT_OK(cache->Insert(k1.AsSlice(), item1.get(), &LRUCacheSecondaryCacheTest::helper_, str1.length())); item1.release(); // Appease clang-analyze "potential memory leak" Cache::Handle* handle; - handle = cache->Lookup(k2.AsSlice(), nullptr, test_item_creator, + handle = cache->Lookup(k2.AsSlice(), nullptr, /*context*/ this, Cache::Priority::LOW, true); ASSERT_EQ(handle, nullptr); handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, false); + /*context*/ this, Cache::Priority::LOW, false); ASSERT_EQ(handle, nullptr); cache.reset(); @@ -1256,18 +1278,18 @@ TEST_F(LRUCacheSecondaryCacheTest, SaveFailTest) { Cache::Handle* handle; handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_fail_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); // This lookup should fail, since k1 demotion would have failed handle = cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_fail_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_EQ(handle, nullptr); // Since k1 didn't get promoted, k2 should still be in cache handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_fail_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); ASSERT_EQ(secondary_cache->num_inserts(), 1u); @@ -1304,16 +1326,16 @@ TEST_F(LRUCacheSecondaryCacheTest, CreateFailTest) { Cache::Handle* handle; SetFailCreate(true); handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); // This lookup should fail, since k1 creation would have failed handle = cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_EQ(handle, nullptr); // Since k1 didn't get promoted, k2 should still be in cache handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); ASSERT_EQ(secondary_cache->num_inserts(), 1u); @@ -1349,19 +1371,19 @@ TEST_F(LRUCacheSecondaryCacheTest, FullCapacityTest) { Cache::Handle* handle; handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); // k1 promotion should fail due to the block cache being at capacity, // but the lookup should still succeed Cache::Handle* handle2; handle2 = cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle2, nullptr); // Since k1 didn't get inserted, k2 should still be in cache cache->Release(handle); cache->Release(handle2); handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, true); + /*context*/ this, Cache::Priority::LOW, true); ASSERT_NE(handle, nullptr); cache->Release(handle); ASSERT_EQ(secondary_cache->num_inserts(), 1u); @@ -1838,7 +1860,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) { for (int i = 0; i < 6; ++i) { results.emplace_back(cache->Lookup( ock.WithOffset(i).AsSlice(), &LRUCacheSecondaryCacheTest::helper_, - test_item_creator, Cache::Priority::LOW, false)); + /*context*/ this, Cache::Priority::LOW, false)); } cache->WaitAll(results); for (int i = 0; i < 6; ++i) { @@ -1964,26 +1986,18 @@ class LRUCacheWithStat : public LRUCache { } ~LRUCacheWithStat() {} - Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, - Handle** handle, Priority priority) override { - insert_count_++; - return LRUCache::Insert(key, value, charge, deleter, handle, priority); - } - Status Insert(const Slice& key, void* value, const CacheItemHelper* helper, - size_t charge, Handle** handle = nullptr, + Status Insert(const Slice& key, Cache::ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, Priority priority = Priority::LOW) override { insert_count_++; return LRUCache::Insert(key, value, helper, charge, handle, priority); } - Handle* Lookup(const Slice& key, Statistics* stats) override { - lookup_count_++; - return LRUCache::Lookup(key, stats); - } Handle* Lookup(const Slice& key, const CacheItemHelper* helper, - const CreateCallback& create_cb, Priority priority, bool wait, + CreateContext* create_context, Priority priority, bool wait, Statistics* stats = nullptr) override { lookup_count_++; - return LRUCache::Lookup(key, helper, create_cb, priority, wait, stats); + return LRUCache::Lookup(key, helper, create_context, priority, wait, stats); } uint32_t GetInsertCount() { return insert_count_; } diff --git a/cache/secondary_cache.cc b/cache/secondary_cache.cc index 84352db71b..eb4972f8f0 100644 --- a/cache/secondary_cache.cc +++ b/cache/secondary_cache.cc @@ -11,20 +11,29 @@ namespace ROCKSDB_NAMESPACE { namespace { -size_t SliceSize(void* obj) { return static_cast(obj)->size(); } +void NoopDelete(Cache::ObjectPtr, MemoryAllocator*) {} -Status SliceSaveTo(void* from_obj, size_t from_offset, size_t length, - void* out) { +size_t SliceSize(Cache::ObjectPtr obj) { + return static_cast(obj)->size(); +} + +Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, size_t length, + char* out) { const Slice& slice = *static_cast(from_obj); std::memcpy(out, slice.data() + from_offset, length); return Status::OK(); } +Status FailCreate(const Slice&, Cache::CreateContext*, MemoryAllocator*, + Cache::ObjectPtr*, size_t*) { + return Status::NotSupported("Only for dumping data into SecondaryCache"); +} + } // namespace Status SecondaryCache::InsertSaved(const Slice& key, const Slice& saved) { - static Cache::CacheItemHelper helper{ - &SliceSize, &SliceSaveTo, GetNoopDeleterForRole()}; + static Cache::CacheItemHelper helper{CacheEntryRole::kMisc, &NoopDelete, + &SliceSize, &SliceSaveTo, &FailCreate}; // NOTE: depends on Insert() being synchronous, not keeping pointer `&saved` return Insert(key, const_cast(&saved), &helper); } diff --git a/cache/sharded_cache.h b/cache/sharded_cache.h index e3271cc7bd..65764579fe 100644 --- a/cache/sharded_cache.h +++ b/cache/sharded_cache.h @@ -49,16 +49,12 @@ class CacheShardBase { HashCref GetHash() const; ... }; - Status Insert(const Slice& key, HashCref hash, void* value, size_t charge, - DeleterFn deleter, HandleImpl** handle, - Cache::Priority priority) = 0; - Status Insert(const Slice& key, HashCref hash, void* value, + Status Insert(const Slice& key, HashCref hash, Cache::ObjectPtr value, const Cache::CacheItemHelper* helper, size_t charge, HandleImpl** handle, Cache::Priority priority) = 0; - HandleImpl* Lookup(const Slice& key, HashCref hash) = 0; HandleImpl* Lookup(const Slice& key, HashCref hash, const Cache::CacheItemHelper* helper, - const Cache::CreateCallback& create_cb, + Cache::CreateContext* create_context, Cache::Priority priority, bool wait, Statistics* stats) = 0; bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref) = 0; @@ -77,8 +73,9 @@ class CacheShardBase { // *state == 0 and implementation sets *state = SIZE_MAX to indicate // completion. void ApplyToSomeEntries( - const std::function& callback, + const std::function& callback, size_t average_entries_per_lock, size_t* state) = 0; void EraseUnRefEntries() = 0; */ @@ -172,36 +169,24 @@ class ShardedCache : public ShardedCacheBase { [s_c_l](CacheShard* cs) { cs->SetStrictCapacityLimit(s_c_l); }); } - Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter, - Handle** handle, Priority priority) override { - HashVal hash = CacheShard::ComputeHash(key); - auto h_out = reinterpret_cast(handle); - return GetShard(hash).Insert(key, hash, value, charge, deleter, h_out, - priority); - } - Status Insert(const Slice& key, void* value, const CacheItemHelper* helper, - size_t charge, Handle** handle = nullptr, + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, Priority priority = Priority::LOW) override { - if (!helper) { - return Status::InvalidArgument(); - } + assert(helper); HashVal hash = CacheShard::ComputeHash(key); auto h_out = reinterpret_cast(handle); return GetShard(hash).Insert(key, hash, value, helper, charge, h_out, priority); } - Handle* Lookup(const Slice& key, Statistics* /*stats*/) override { - HashVal hash = CacheShard::ComputeHash(key); - HandleImpl* result = GetShard(hash).Lookup(key, hash); - return reinterpret_cast(result); - } - Handle* Lookup(const Slice& key, const CacheItemHelper* helper, - const CreateCallback& create_cb, Priority priority, bool wait, + Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr, + CreateContext* create_context = nullptr, + Priority priority = Priority::LOW, bool wait = true, Statistics* stats = nullptr) override { HashVal hash = CacheShard::ComputeHash(key); - HandleImpl* result = GetShard(hash).Lookup(key, hash, helper, create_cb, - priority, wait, stats); + HandleImpl* result = GetShard(hash).Lookup( + key, hash, helper, create_context, priority, wait, stats); return reinterpret_cast(result); } @@ -244,8 +229,8 @@ class ShardedCache : public ShardedCacheBase { return SumOverShards2(&CacheShard::GetTableAddressCount); } void ApplyToAllEntries( - const std::function& callback, + const std::function& callback, const ApplyToAllEntriesOptions& opts) override { uint32_t num_shards = GetNumShards(); // Iterate over part of each shard, rotating between shards, to diff --git a/cache/typed_cache.h b/cache/typed_cache.h new file mode 100644 index 0000000000..76c82b4a05 --- /dev/null +++ b/cache/typed_cache.h @@ -0,0 +1,339 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// APIs for accessing Cache in a type-safe and convenient way. Cache is kept +// at a low, thin level of abstraction so that different implementations can +// be plugged in, but these wrappers provide clean, convenient access to the +// most common operations. +// +// A number of template classes are needed for sharing common structure. The +// key classes are these: +// +// * PlaceholderCacheInterface - Used for making cache reservations, with +// entries that have a charge but no value. +// * BasicTypedCacheInterface - Used for primary cache storage of +// objects of type TValue. +// * FullTypedCacheHelper - Used for secondary cache +// compatible storage of objects of type TValue. +// * For each of these, there's a "Shared" version +// (e.g. FullTypedSharedCacheInterface) that holds a shared_ptr to the Cache, +// rather than assuming external ownership by holding only a raw `Cache*`. + +#pragma once + +#include +#include +#include +#include + +#include "cache/cache_helpers.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/cache.h" + +namespace ROCKSDB_NAMESPACE { + +// For future consideration: +// * Pass in value to Insert with std::unique_ptr& to simplify ownership +// transfer logic in callers +// * Make key type a template parameter (e.g. useful for table cache) +// * Closer integration with CacheHandleGuard (opt-in, so not always +// paying the extra overhead) + +#define CACHE_TYPE_DEFS() \ + using Priority = Cache::Priority; \ + using Handle = Cache::Handle; \ + using ObjectPtr = Cache::ObjectPtr; \ + using CreateContext = Cache::CreateContext; \ + using CacheItemHelper = Cache::CacheItemHelper /* caller ; */ + +template +class BaseCacheInterface { + public: + CACHE_TYPE_DEFS(); + + /*implicit*/ BaseCacheInterface(CachePtr cache) : cache_(std::move(cache)) {} + + inline void Release(Handle* handle) { cache_->Release(handle); } + + inline void ReleaseAndEraseIfLastRef(Handle* handle) { + cache_->Release(handle, /*erase_if_last_ref*/ true); + } + + inline void RegisterReleaseAsCleanup(Handle* handle, Cleanable& cleanable) { + cleanable.RegisterCleanup(&ReleaseCacheHandleCleanup, get(), handle); + } + + inline Cache* get() const { return &*cache_; } + + explicit inline operator bool() const noexcept { return cache_ != nullptr; } + + protected: + CachePtr cache_; +}; + +// PlaceholderCacheInterface - Used for making cache reservations, with +// entries that have a charge but no value. CacheEntryRole is required as +// a template parameter. +template +class PlaceholderCacheInterface : public BaseCacheInterface { + public: + CACHE_TYPE_DEFS(); + using BaseCacheInterface::BaseCacheInterface; + + inline Status Insert(const Slice& key, size_t charge, Handle** handle) { + return this->cache_->Insert(key, /*value=*/nullptr, &kHelper, charge, + handle); + } + + static constexpr Cache::CacheItemHelper kHelper{kRole}; +}; + +template +using PlaceholderSharedCacheInterface = + PlaceholderCacheInterface>; + +template +class BasicTypedCacheHelperFns { + public: + CACHE_TYPE_DEFS(); + // E.g. char* for char[] + using TValuePtr = std::remove_extent_t*; + + protected: + inline static ObjectPtr UpCastValue(TValuePtr value) { return value; } + inline static TValuePtr DownCastValue(ObjectPtr value) { + return static_cast(value); + } + + static void Delete(ObjectPtr value, MemoryAllocator* allocator) { + // FIXME: Currently, no callers actually allocate the ObjectPtr objects + // using the custom allocator, just subobjects that keep a reference to + // the allocator themselves (with CacheAllocationPtr). + if (/*DISABLED*/ false && allocator) { + if constexpr (std::is_destructible_v) { + DownCastValue(value)->~TValue(); + } + allocator->Deallocate(value); + } else { + // Like delete but properly handles TValue=char[] etc. + std::default_delete{}(DownCastValue(value)); + } + } +}; + +// In its own class to try to minimize the number of distinct CacheItemHelper +// instances (e.g. don't vary by CachePtr) +template +class BasicTypedCacheHelper : public BasicTypedCacheHelperFns { + public: + static constexpr Cache::CacheItemHelper kBasicHelper{ + kRole, &BasicTypedCacheHelper::Delete}; +}; + +// BasicTypedCacheInterface - Used for primary cache storage of objects of +// type TValue, which can be cleaned up with std::default_delete. The +// role is provided by TValue::kCacheEntryRole or given in an optional +// template parameter. +template +class BasicTypedCacheInterface : public BaseCacheInterface, + public BasicTypedCacheHelper { + public: + CACHE_TYPE_DEFS(); + using typename BasicTypedCacheHelperFns::TValuePtr; + struct TypedHandle : public Handle {}; + using BasicTypedCacheHelper::kBasicHelper; + // ctor + using BaseCacheInterface::BaseCacheInterface; + + inline Status Insert(const Slice& key, TValuePtr value, size_t charge, + TypedHandle** handle = nullptr, + Priority priority = Priority::LOW) { + auto untyped_handle = reinterpret_cast(handle); + return this->cache_->Insert( + key, BasicTypedCacheHelperFns::UpCastValue(value), + &kBasicHelper, charge, untyped_handle, priority); + } + + inline TypedHandle* Lookup(const Slice& key, Statistics* stats = nullptr) { + return reinterpret_cast( + this->cache_->BasicLookup(key, stats)); + } + + inline CacheHandleGuard Guard(TypedHandle* handle) { + if (handle) { + return CacheHandleGuard(&*this->cache_, handle); + } else { + return {}; + } + } + + inline std::shared_ptr SharedGuard(TypedHandle* handle) { + if (handle) { + return MakeSharedCacheHandleGuard(&*this->cache_, handle); + } else { + return {}; + } + } + + inline TValuePtr Value(TypedHandle* handle) { + return BasicTypedCacheHelperFns::DownCastValue( + this->cache_->Value(handle)); + } +}; + +// BasicTypedSharedCacheInterface - Like BasicTypedCacheInterface but with a +// shared_ptr for keeping Cache alive. +template +using BasicTypedSharedCacheInterface = + BasicTypedCacheInterface>; + +// TValue must implement ContentSlice() and ~TValue +// TCreateContext must implement Create(std::unique_ptr*, ...) +template +class FullTypedCacheHelperFns : public BasicTypedCacheHelperFns { + public: + CACHE_TYPE_DEFS(); + + protected: + using typename BasicTypedCacheHelperFns::TValuePtr; + using BasicTypedCacheHelperFns::DownCastValue; + using BasicTypedCacheHelperFns::UpCastValue; + + static size_t Size(ObjectPtr v) { + TValuePtr value = DownCastValue(v); + auto slice = value->ContentSlice(); + return slice.size(); + } + + static Status SaveTo(ObjectPtr v, size_t from_offset, size_t length, + char* out) { + TValuePtr value = DownCastValue(v); + auto slice = value->ContentSlice(); + assert(from_offset < slice.size()); + assert(from_offset + length <= slice.size()); + std::copy_n(slice.data() + from_offset, length, out); + return Status::OK(); + } + + static Status Create(const Slice& data, CreateContext* context, + MemoryAllocator* allocator, ObjectPtr* out_obj, + size_t* out_charge) { + std::unique_ptr value = nullptr; + if constexpr (sizeof(TCreateContext) > 0) { + TCreateContext* tcontext = static_cast(context); + tcontext->Create(&value, out_charge, data, allocator); + } else { + TCreateContext::Create(&value, out_charge, data, allocator); + } + *out_obj = UpCastValue(value.release()); + return Status::OK(); + } +}; + +// In its own class to try to minimize the number of distinct CacheItemHelper +// instances (e.g. don't vary by CachePtr) +template +class FullTypedCacheHelper + : public FullTypedCacheHelperFns { + public: + static constexpr Cache::CacheItemHelper kFullHelper{ + kRole, &FullTypedCacheHelper::Delete, &FullTypedCacheHelper::Size, + &FullTypedCacheHelper::SaveTo, &FullTypedCacheHelper::Create}; +}; + +// FullTypedCacheHelper - Used for secondary cache compatible storage of +// objects of type TValue. In addition to BasicTypedCacheInterface constraints, +// we require TValue::ContentSlice() to return persistable data. This +// simplifies usage for the normal case of simple secondary cache compatibility +// (can give you a Slice to the data already in memory). In addition to +// TCreateContext performing the role of Cache::CreateContext, it is also +// expected to provide a function Create(std::unique_ptr* value, +// size_t* out_charge, const Slice& data, MemoryAllocator* allocator) for +// creating new TValue. +template +class FullTypedCacheInterface + : public BasicTypedCacheInterface, + public FullTypedCacheHelper { + public: + CACHE_TYPE_DEFS(); + using typename BasicTypedCacheInterface::TypedHandle; + using typename BasicTypedCacheHelperFns::TValuePtr; + using BasicTypedCacheHelper::kBasicHelper; + using FullTypedCacheHelper::kFullHelper; + using BasicTypedCacheHelperFns::UpCastValue; + using BasicTypedCacheHelperFns::DownCastValue; + // ctor + using BasicTypedCacheInterface::BasicTypedCacheInterface; + + // Insert with SecondaryCache compatibility (subject to CacheTier). + // (Basic Insert() also inherited.) + inline Status InsertFull( + const Slice& key, TValuePtr value, size_t charge, + TypedHandle** handle = nullptr, Priority priority = Priority::LOW, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { + auto untyped_handle = reinterpret_cast(handle); + auto helper = lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier + ? &kFullHelper + : &kBasicHelper; + return this->cache_->Insert(key, UpCastValue(value), helper, charge, + untyped_handle, priority); + } + + // Like SecondaryCache::InsertSaved, with SecondaryCache compatibility + // (subject to CacheTier). + inline Status InsertSaved( + const Slice& key, const Slice& data, TCreateContext* create_context, + Priority priority = Priority::LOW, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier, + size_t* out_charge = nullptr) { + ObjectPtr value; + size_t charge; + Status st = kFullHelper.create_cb(data, create_context, + this->cache_->memory_allocator(), &value, + &charge); + if (out_charge) { + *out_charge = charge; + } + if (st.ok()) { + st = InsertFull(key, DownCastValue(value), charge, nullptr /*handle*/, + priority, lowest_used_cache_tier); + } else { + kFullHelper.del_cb(value, this->cache_->memory_allocator()); + } + return st; + } + + // Lookup with SecondaryCache support (subject to CacheTier). + // (Basic Lookup() also inherited.) + inline TypedHandle* LookupFull( + const Slice& key, TCreateContext* create_context = nullptr, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { + if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + return reinterpret_cast(this->cache_->Lookup( + key, &kFullHelper, create_context, priority, wait, stats)); + } else { + return BasicTypedCacheInterface::Lookup(key, + stats); + } + } +}; + +// FullTypedSharedCacheInterface - Like FullTypedCacheInterface but with a +// shared_ptr for keeping Cache alive. +template +using FullTypedSharedCacheInterface = + FullTypedCacheInterface>; + +#undef CACHE_TYPE_DEFS + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_contents.cc b/db/blob/blob_contents.cc index 9015609e7f..8b793c5d26 100644 --- a/db/blob/blob_contents.cc +++ b/db/blob/blob_contents.cc @@ -13,12 +13,6 @@ namespace ROCKSDB_NAMESPACE { -std::unique_ptr BlobContents::Create( - CacheAllocationPtr&& allocation, size_t size) { - return std::unique_ptr( - new BlobContents(std::move(allocation), size)); -} - size_t BlobContents::ApproximateMemoryUsage() const { size_t usage = 0; @@ -45,46 +39,4 @@ size_t BlobContents::ApproximateMemoryUsage() const { return usage; } -size_t BlobContents::SizeCallback(void* obj) { - assert(obj); - - return static_cast(obj)->size(); -} - -Status BlobContents::SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out) { - assert(from_obj); - - const BlobContents* buf = static_cast(from_obj); - assert(buf->size() >= from_offset + length); - - memcpy(out, buf->data().data() + from_offset, length); - - return Status::OK(); -} - -Cache::CacheItemHelper* BlobContents::GetCacheItemHelper() { - static Cache::CacheItemHelper cache_helper( - &SizeCallback, &SaveToCallback, - GetCacheEntryDeleterForRole()); - - return &cache_helper; -} - -Status BlobContents::CreateCallback(CacheAllocationPtr&& allocation, - const void* buf, size_t size, - void** out_obj, size_t* charge) { - assert(allocation); - - memcpy(allocation.get(), buf, size); - - std::unique_ptr obj = Create(std::move(allocation), size); - BlobContents* const contents = obj.release(); - - *out_obj = contents; - *charge = contents->ApproximateMemoryUsage(); - - return Status::OK(); -} - } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_contents.h b/db/blob/blob_contents.h index 9b7c5b9696..18ed27c692 100644 --- a/db/blob/blob_contents.h +++ b/db/blob/blob_contents.h @@ -18,8 +18,8 @@ namespace ROCKSDB_NAMESPACE { // A class representing a single uncompressed value read from a blob file. class BlobContents { public: - static std::unique_ptr Create(CacheAllocationPtr&& allocation, - size_t size); + BlobContents(CacheAllocationPtr&& allocation, size_t size) + : allocation_(std::move(allocation)), data_(allocation_.get(), size) {} BlobContents(const BlobContents&) = delete; BlobContents& operator=(const BlobContents&) = delete; @@ -34,23 +34,26 @@ class BlobContents { size_t ApproximateMemoryUsage() const; - // Callbacks for secondary cache - static size_t SizeCallback(void* obj); - - static Status SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out); - - static Cache::CacheItemHelper* GetCacheItemHelper(); - - static Status CreateCallback(CacheAllocationPtr&& allocation, const void* buf, - size_t size, void** out_obj, size_t* charge); + // For TypedCacheInterface + const Slice& ContentSlice() const { return data_; } + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kBlobValue; private: - BlobContents(CacheAllocationPtr&& allocation, size_t size) - : allocation_(std::move(allocation)), data_(allocation_.get(), size) {} - CacheAllocationPtr allocation_; Slice data_; }; +class BlobContentsCreator : public Cache::CreateContext { + public: + static void Create(std::unique_ptr* out, size_t* out_charge, + const Slice& contents, MemoryAllocator* alloc) { + auto raw = new BlobContents(AllocateAndCopyBlock(contents, alloc), + contents.size()); + out->reset(raw); + if (out_charge) { + *out_charge = raw->ApproximateMemoryUsage(); + } + } +}; + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 5e0e7f6cb4..952a5676bf 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -13,6 +13,7 @@ #include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" #include "db/blob/blob_log_writer.h" +#include "db/blob/blob_source.h" #include "db/event_helpers.h" #include "db/version_set.h" #include "file/filename.h" @@ -393,7 +394,7 @@ Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob, uint64_t blob_offset) const { Status s = Status::OK(); - auto blob_cache = immutable_options_->blob_cache; + BlobSource::SharedCacheInterface blob_cache{immutable_options_->blob_cache}; auto statistics = immutable_options_->statistics.get(); bool warm_cache = prepopulate_blob_cache_ == PrepopulateBlobCache::kFlushOnly && @@ -407,34 +408,12 @@ Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob, const Cache::Priority priority = Cache::Priority::BOTTOM; - // Objects to be put into the cache have to be heap-allocated and - // self-contained, i.e. own their contents. The Cache has to be able to - // take unique ownership of them. - CacheAllocationPtr allocation = - AllocateBlock(blob.size(), blob_cache->memory_allocator()); - memcpy(allocation.get(), blob.data(), blob.size()); - std::unique_ptr buf = - BlobContents::Create(std::move(allocation), blob.size()); - - Cache::CacheItemHelper* const cache_item_helper = - BlobContents::GetCacheItemHelper(); - assert(cache_item_helper); - - if (immutable_options_->lowest_used_cache_tier == - CacheTier::kNonVolatileBlockTier) { - s = blob_cache->Insert(key, buf.get(), cache_item_helper, - buf->ApproximateMemoryUsage(), - nullptr /* cache_handle */, priority); - } else { - s = blob_cache->Insert(key, buf.get(), buf->ApproximateMemoryUsage(), - cache_item_helper->del_cb, - nullptr /* cache_handle */, priority); - } + s = blob_cache.InsertSaved(key, blob, nullptr /*context*/, priority, + immutable_options_->lowest_used_cache_tier); if (s.ok()) { RecordTick(statistics, BLOB_DB_CACHE_ADD); - RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, buf->size()); - buf.release(); + RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, blob.size()); } else { RecordTick(statistics, BLOB_DB_CACHE_ADD_FAILURES); } diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc index 1a6cdf6880..19757946d6 100644 --- a/db/blob/blob_file_cache.cc +++ b/db/blob/blob_file_cache.cc @@ -42,13 +42,13 @@ Status BlobFileCache::GetBlobFileReader( assert(blob_file_reader); assert(blob_file_reader->IsEmpty()); - const Slice key = GetSlice(&blob_file_number); + const Slice key = GetSliceForKey(&blob_file_number); assert(cache_); - Cache::Handle* handle = cache_->Lookup(key); + TypedHandle* handle = cache_.Lookup(key); if (handle) { - *blob_file_reader = CacheHandleGuard(cache_, handle); + *blob_file_reader = cache_.Guard(handle); return Status::OK(); } @@ -57,9 +57,9 @@ Status BlobFileCache::GetBlobFileReader( // Check again while holding mutex MutexLock lock(mutex_.get(key)); - handle = cache_->Lookup(key); + handle = cache_.Lookup(key); if (handle) { - *blob_file_reader = CacheHandleGuard(cache_, handle); + *blob_file_reader = cache_.Guard(handle); return Status::OK(); } @@ -84,8 +84,7 @@ Status BlobFileCache::GetBlobFileReader( { constexpr size_t charge = 1; - const Status s = cache_->Insert(key, reader.get(), charge, - &DeleteCacheEntry, &handle); + const Status s = cache_.Insert(key, reader.get(), charge, &handle); if (!s.ok()) { RecordTick(statistics, NO_FILE_ERRORS); return s; @@ -94,7 +93,7 @@ Status BlobFileCache::GetBlobFileReader( reader.release(); - *blob_file_reader = CacheHandleGuard(cache_, handle); + *blob_file_reader = cache_.Guard(handle); return Status::OK(); } diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h index 8eec05f184..6281897d60 100644 --- a/db/blob/blob_file_cache.h +++ b/db/blob/blob_file_cache.h @@ -7,7 +7,8 @@ #include -#include "cache/cache_helpers.h" +#include "cache/typed_cache.h" +#include "db/blob/blob_file_reader.h" #include "rocksdb/rocksdb_namespace.h" #include "util/mutexlock.h" @@ -18,7 +19,6 @@ struct ImmutableOptions; struct FileOptions; class HistogramImpl; class Status; -class BlobFileReader; class Slice; class IOTracer; @@ -36,7 +36,10 @@ class BlobFileCache { CacheHandleGuard* blob_file_reader); private: - Cache* cache_; + using CacheInterface = + BasicTypedCacheInterface; + using TypedHandle = CacheInterface::TypedHandle; + CacheInterface cache_; // Note: mutex_ below is used to guard against multiple threads racing to open // the same file. Striped mutex_; diff --git a/db/blob/blob_file_reader.cc b/db/blob/blob_file_reader.cc index a4eabb6059..da7f2bb12e 100644 --- a/db/blob/blob_file_reader.cc +++ b/db/blob/blob_file_reader.cc @@ -569,12 +569,7 @@ Status BlobFileReader::UncompressBlobIfNeeded( assert(result); if (compression_type == kNoCompression) { - CacheAllocationPtr allocation = - AllocateBlock(value_slice.size(), allocator); - memcpy(allocation.get(), value_slice.data(), value_slice.size()); - - *result = BlobContents::Create(std::move(allocation), value_slice.size()); - + BlobContentsCreator::Create(result, nullptr, value_slice, allocator); return Status::OK(); } @@ -602,7 +597,7 @@ Status BlobFileReader::UncompressBlobIfNeeded( return Status::Corruption("Unable to uncompress blob"); } - *result = BlobContents::Create(std::move(output), uncompressed_size); + result->reset(new BlobContents(std::move(output), uncompressed_size)); return Status::OK(); } diff --git a/db/blob/blob_source.cc b/db/blob/blob_source.cc index bfade2507e..19cfb1f89a 100644 --- a/db/blob/blob_source.cc +++ b/db/blob/blob_source.cc @@ -36,8 +36,8 @@ BlobSource::BlobSource(const ImmutableOptions* immutable_options, if (bbto && bbto->cache_usage_options.options_overrides.at(CacheEntryRole::kBlobCache) .charged == CacheEntryRoleOptions::Decision::kEnabled) { - blob_cache_ = std::make_shared(immutable_options->blob_cache, - bbto->block_cache); + blob_cache_ = SharedCacheInterface{std::make_shared( + immutable_options->blob_cache, bbto->block_cache)}; } #endif // ROCKSDB_LITE } @@ -82,9 +82,8 @@ Status BlobSource::PutBlobIntoCache( assert(cached_blob); assert(cached_blob->IsEmpty()); - Cache::Handle* cache_handle = nullptr; + TypedHandle* cache_handle = nullptr; const Status s = InsertEntryIntoCache(cache_key, blob->get(), - (*blob)->ApproximateMemoryUsage(), &cache_handle, Cache::Priority::BOTTOM); if (s.ok()) { blob->release(); @@ -106,26 +105,10 @@ Status BlobSource::PutBlobIntoCache( return s; } -Cache::Handle* BlobSource::GetEntryFromCache(const Slice& key) const { - Cache::Handle* cache_handle = nullptr; - - if (lowest_used_cache_tier_ == CacheTier::kNonVolatileBlockTier) { - Cache::CreateCallback create_cb = - [allocator = blob_cache_->memory_allocator()]( - const void* buf, size_t size, void** out_obj, - size_t* charge) -> Status { - return BlobContents::CreateCallback(AllocateBlock(size, allocator), buf, - size, out_obj, charge); - }; - - cache_handle = blob_cache_->Lookup(key, BlobContents::GetCacheItemHelper(), - create_cb, Cache::Priority::BOTTOM, - true /* wait_for_cache */, statistics_); - } else { - cache_handle = blob_cache_->Lookup(key, statistics_); - } - - return cache_handle; +BlobSource::TypedHandle* BlobSource::GetEntryFromCache(const Slice& key) const { + return blob_cache_.LookupFull( + key, nullptr /* context */, Cache::Priority::BOTTOM, + true /* wait_for_cache */, statistics_, lowest_used_cache_tier_); } void BlobSource::PinCachedBlob(CacheHandleGuard* cached_blob, @@ -166,24 +149,11 @@ void BlobSource::PinOwnedBlob(std::unique_ptr* owned_blob, } Status BlobSource::InsertEntryIntoCache(const Slice& key, BlobContents* value, - size_t charge, - Cache::Handle** cache_handle, + TypedHandle** cache_handle, Cache::Priority priority) const { - Status s; - - Cache::CacheItemHelper* const cache_item_helper = - BlobContents::GetCacheItemHelper(); - assert(cache_item_helper); - - if (lowest_used_cache_tier_ == CacheTier::kNonVolatileBlockTier) { - s = blob_cache_->Insert(key, value, cache_item_helper, charge, cache_handle, - priority); - } else { - s = blob_cache_->Insert(key, value, charge, cache_item_helper->del_cb, - cache_handle, priority); - } - - return s; + return blob_cache_.InsertFull(key, value, value->ApproximateMemoryUsage(), + cache_handle, priority, + lowest_used_cache_tier_); } Status BlobSource::GetBlob(const ReadOptions& read_options, @@ -252,9 +222,10 @@ Status BlobSource::GetBlob(const ReadOptions& read_options, return Status::Corruption("Compression type mismatch when reading blob"); } - MemoryAllocator* const allocator = (blob_cache_ && read_options.fill_cache) - ? blob_cache_->memory_allocator() - : nullptr; + MemoryAllocator* const allocator = + (blob_cache_ && read_options.fill_cache) + ? blob_cache_.get()->memory_allocator() + : nullptr; uint64_t read_size = 0; s = blob_file_reader.GetValue()->GetBlob( @@ -418,9 +389,10 @@ void BlobSource::MultiGetBlobFromOneFile(const ReadOptions& read_options, assert(blob_file_reader.GetValue()); - MemoryAllocator* const allocator = (blob_cache_ && read_options.fill_cache) - ? blob_cache_->memory_allocator() - : nullptr; + MemoryAllocator* const allocator = + (blob_cache_ && read_options.fill_cache) + ? blob_cache_.get()->memory_allocator() + : nullptr; blob_file_reader.GetValue()->MultiGetBlob(read_options, allocator, _blob_reqs, &_bytes_read); diff --git a/db/blob/blob_source.h b/db/blob/blob_source.h index 2ed296eeb4..cdc2187478 100644 --- a/db/blob/blob_source.h +++ b/db/blob/blob_source.h @@ -8,8 +8,9 @@ #include #include -#include "cache/cache_helpers.h" #include "cache/cache_key.h" +#include "cache/typed_cache.h" +#include "db/blob/blob_contents.h" #include "db/blob/blob_file_cache.h" #include "db/blob/blob_read_request.h" #include "rocksdb/cache.h" @@ -23,7 +24,6 @@ struct ImmutableOptions; class Status; class FilePrefetchBuffer; class Slice; -class BlobContents; // BlobSource is a class that provides universal access to blobs, regardless of // whether they are in the blob cache, secondary cache, or (remote) storage. @@ -106,6 +106,14 @@ class BlobSource { bool TEST_BlobInCache(uint64_t file_number, uint64_t file_size, uint64_t offset, size_t* charge = nullptr) const; + // For TypedSharedCacheInterface + void Create(BlobContents** out, const char* buf, size_t size, + MemoryAllocator* alloc); + + using SharedCacheInterface = + FullTypedSharedCacheInterface; + using TypedHandle = SharedCacheInterface::TypedHandle; + private: Status GetBlobFromCache(const Slice& cache_key, CacheHandleGuard* cached_blob) const; @@ -120,10 +128,10 @@ class BlobSource { static void PinOwnedBlob(std::unique_ptr* owned_blob, PinnableSlice* value); - Cache::Handle* GetEntryFromCache(const Slice& key) const; + TypedHandle* GetEntryFromCache(const Slice& key) const; Status InsertEntryIntoCache(const Slice& key, BlobContents* value, - size_t charge, Cache::Handle** cache_handle, + TypedHandle** cache_handle, Cache::Priority priority) const; inline CacheKey GetCacheKey(uint64_t file_number, uint64_t /*file_size*/, @@ -141,7 +149,7 @@ class BlobSource { BlobFileCache* blob_file_cache_; // A cache to store uncompressed blobs. - std::shared_ptr blob_cache_; + mutable SharedCacheInterface blob_cache_; // The control option of how the cache tiers will be used. Currently rocksdb // support block/blob cache (volatile tier) and secondary cache (this tier diff --git a/db/blob/blob_source_test.cc b/db/blob/blob_source_test.cc index a85ed86462..4a1ba84eea 100644 --- a/db/blob/blob_source_test.cc +++ b/db/blob/blob_source_test.cc @@ -1150,15 +1150,6 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { auto blob_cache = options_.blob_cache; auto secondary_cache = lru_cache_opts_.secondary_cache; - Cache::CreateCallback create_cb = [](const void* buf, size_t size, - void** out_obj, - size_t* charge) -> Status { - CacheAllocationPtr allocation(new char[size]); - - return BlobContents::CreateCallback(std::move(allocation), buf, size, - out_obj, charge); - }; - { // GetBlob std::vector values(keys.size()); @@ -1219,14 +1210,15 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { { CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[0]); const Slice key0 = cache_key.AsSlice(); - auto handle0 = blob_cache->Lookup(key0, statistics); + auto handle0 = blob_cache->BasicLookup(key0, statistics); ASSERT_EQ(handle0, nullptr); // key0's item should be in the secondary cache. bool is_in_sec_cache = false; - auto sec_handle0 = - secondary_cache->Lookup(key0, create_cb, true, - /*advise_erase=*/true, is_in_sec_cache); + auto sec_handle0 = secondary_cache->Lookup( + key0, &BlobSource::SharedCacheInterface::kFullHelper, + /*context*/ nullptr, true, + /*advise_erase=*/true, is_in_sec_cache); ASSERT_FALSE(is_in_sec_cache); ASSERT_NE(sec_handle0, nullptr); ASSERT_TRUE(sec_handle0->IsReady()); @@ -1246,14 +1238,15 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { { CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[1]); const Slice key1 = cache_key.AsSlice(); - auto handle1 = blob_cache->Lookup(key1, statistics); + auto handle1 = blob_cache->BasicLookup(key1, statistics); ASSERT_NE(handle1, nullptr); blob_cache->Release(handle1); bool is_in_sec_cache = false; - auto sec_handle1 = - secondary_cache->Lookup(key1, create_cb, true, - /*advise_erase=*/true, is_in_sec_cache); + auto sec_handle1 = secondary_cache->Lookup( + key1, &BlobSource::SharedCacheInterface::kFullHelper, + /*context*/ nullptr, true, + /*advise_erase=*/true, is_in_sec_cache); ASSERT_FALSE(is_in_sec_cache); ASSERT_EQ(sec_handle1, nullptr); @@ -1276,7 +1269,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { // key0 should be in the primary cache. CacheKey cache_key0 = base_cache_key.WithOffset(blob_offsets[0]); const Slice key0 = cache_key0.AsSlice(); - auto handle0 = blob_cache->Lookup(key0, statistics); + auto handle0 = blob_cache->BasicLookup(key0, statistics); ASSERT_NE(handle0, nullptr); auto value = static_cast(blob_cache->Value(handle0)); ASSERT_NE(value, nullptr); @@ -1286,12 +1279,12 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { // key1 is not in the primary cache and is in the secondary cache. CacheKey cache_key1 = base_cache_key.WithOffset(blob_offsets[1]); const Slice key1 = cache_key1.AsSlice(); - auto handle1 = blob_cache->Lookup(key1, statistics); + auto handle1 = blob_cache->BasicLookup(key1, statistics); ASSERT_EQ(handle1, nullptr); // erase key0 from the primary cache. blob_cache->Erase(key0); - handle0 = blob_cache->Lookup(key0, statistics); + handle0 = blob_cache->BasicLookup(key0, statistics); ASSERT_EQ(handle0, nullptr); // key1 promotion should succeed due to the primary cache being empty. we @@ -1307,7 +1300,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { // in the secondary cache. So, the primary cache's Lookup() without // secondary cache support cannot see it. (NOTE: The dummy handle used // to be a leaky abstraction but not anymore.) - handle1 = blob_cache->Lookup(key1, statistics); + handle1 = blob_cache->BasicLookup(key1, statistics); ASSERT_EQ(handle1, nullptr); // But after another access, it is promoted to primary cache @@ -1315,7 +1308,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { blob_offsets[1])); // And Lookup() can find it (without secondary cache support) - handle1 = blob_cache->Lookup(key1, statistics); + handle1 = blob_cache->BasicLookup(key1, statistics); ASSERT_NE(handle1, nullptr); ASSERT_NE(blob_cache->Value(handle1), nullptr); blob_cache->Release(handle1); diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index a28ac2b885..0fa53fead4 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -3537,24 +3537,27 @@ class DBBasicTestMultiGet : public DBTestBase { const char* Name() const override { return "MyBlockCache"; } - using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), + Status Insert(const Slice& key, Cache::ObjectPtr value, + const CacheItemHelper* helper, size_t charge, Handle** handle = nullptr, Priority priority = Priority::LOW) override { num_inserts_++; - return target_->Insert(key, value, charge, deleter, handle, priority); + return target_->Insert(key, value, helper, charge, handle, priority); } - using Cache::Lookup; - Handle* Lookup(const Slice& key, Statistics* stats = nullptr) override { + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr) override { num_lookups_++; - Handle* handle = target_->Lookup(key, stats); + Handle* handle = + target_->Lookup(key, helper, create_context, priority, wait, stats); if (handle != nullptr) { num_found_++; } return handle; } + int num_lookups() { return num_lookups_; } int num_found() { return num_found_; } diff --git a/db/db_block_cache_test.cc b/db/db_block_cache_test.cc index db80b82cb2..1c45a8aabf 100644 --- a/db/db_block_cache_test.cc +++ b/db/db_block_cache_test.cc @@ -14,6 +14,7 @@ #include "cache/cache_entry_roles.h" #include "cache/cache_key.h" #include "cache/lru_cache.h" +#include "cache/typed_cache.h" #include "db/column_family.h" #include "db/db_impl/db_impl.h" #include "db/db_test_util.h" @@ -365,9 +366,7 @@ class PersistentCacheFromCache : public PersistentCache { } std::unique_ptr copy{new char[size]}; std::copy_n(data, size, copy.get()); - Status s = cache_->Insert( - key, copy.get(), size, - GetCacheEntryDeleterForRole()); + Status s = cache_.Insert(key, copy.get(), size); if (s.ok()) { copy.release(); } @@ -376,13 +375,13 @@ class PersistentCacheFromCache : public PersistentCache { Status Lookup(const Slice& key, std::unique_ptr* data, size_t* size) override { - auto handle = cache_->Lookup(key); + auto handle = cache_.Lookup(key); if (handle) { - char* ptr = static_cast(cache_->Value(handle)); - *size = cache_->GetCharge(handle); + char* ptr = cache_.Value(handle); + *size = cache_.get()->GetCharge(handle); data->reset(new char[*size]); std::copy_n(ptr, *size, data->get()); - cache_->Release(handle); + cache_.Release(handle); return Status::OK(); } else { return Status::NotFound(); @@ -395,10 +394,10 @@ class PersistentCacheFromCache : public PersistentCache { std::string GetPrintableOptions() const override { return ""; } - uint64_t NewId() override { return cache_->NewId(); } + uint64_t NewId() override { return cache_.get()->NewId(); } private: - std::shared_ptr cache_; + BasicTypedSharedCacheInterface cache_; bool read_only_; }; @@ -406,8 +405,8 @@ class ReadOnlyCacheWrapper : public CacheWrapper { using CacheWrapper::CacheWrapper; using Cache::Insert; - Status Insert(const Slice& /*key*/, void* /*value*/, size_t /*charge*/, - void (*)(const Slice& key, void* value) /*deleter*/, + Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, + const CacheItemHelper* /*helper*/, size_t /*charge*/, Handle** /*handle*/, Priority /*priority*/) override { return Status::NotSupported(); } @@ -827,16 +826,15 @@ class MockCache : public LRUCache { using ShardedCache::Insert; - Status Insert(const Slice& key, void* value, - const Cache::CacheItemHelper* helper_cb, size_t charge, + Status Insert(const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, size_t charge, Handle** handle, Priority priority) override { - DeleterFn delete_cb = helper_cb->del_cb; if (priority == Priority::LOW) { low_pri_insert_count++; } else { high_pri_insert_count++; } - return LRUCache::Insert(key, value, charge, delete_cb, handle, priority); + return LRUCache::Insert(key, value, helper, charge, handle, priority); } }; @@ -916,7 +914,10 @@ class LookupLiarCache : public CacheWrapper { : CacheWrapper(std::move(target)) {} using Cache::Lookup; - Handle* Lookup(const Slice& key, Statistics* stats) override { + Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr, + CreateContext* create_context = nullptr, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr) override { if (nth_lookup_not_found_ == 1) { nth_lookup_not_found_ = 0; return nullptr; @@ -924,7 +925,8 @@ class LookupLiarCache : public CacheWrapper { if (nth_lookup_not_found_ > 1) { --nth_lookup_not_found_; } - return CacheWrapper::Lookup(key, stats); + return CacheWrapper::Lookup(key, helper, create_context, priority, wait, + stats); } // 1 == next lookup, 2 == after next, etc. @@ -1275,12 +1277,11 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) { } static void ClearCache(Cache* cache) { - auto roles = CopyCacheDeleterRoleMap(); std::deque keys; Cache::ApplyToAllEntriesOptions opts; - auto callback = [&](const Slice& key, void* /*value*/, size_t /*charge*/, - Cache::DeleterFn deleter) { - if (roles.find(deleter) == roles.end()) { + auto callback = [&](const Slice& key, Cache::ObjectPtr, size_t /*charge*/, + const Cache::CacheItemHelper* helper) { + if (helper && helper->role == CacheEntryRole::kMisc) { // Keep the stats collector return; } @@ -1450,14 +1451,13 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { ClearCache(cache.get()); Cache::Handle* h = nullptr; if (strcmp(cache->Name(), "LRUCache") == 0) { - ASSERT_OK(cache->Insert("Fill-it-up", nullptr, capacity + 1, - GetNoopDeleterForRole(), - &h, Cache::Priority::HIGH)); + ASSERT_OK(cache->Insert("Fill-it-up", nullptr, &kNoopCacheItemHelper, + capacity + 1, &h, Cache::Priority::HIGH)); } else { // For ClockCache we use a 16-byte key. - ASSERT_OK(cache->Insert("Fill-it-up-xxxxx", nullptr, capacity + 1, - GetNoopDeleterForRole(), - &h, Cache::Priority::HIGH)); + ASSERT_OK(cache->Insert("Fill-it-up-xxxxx", nullptr, + &kNoopCacheItemHelper, capacity + 1, &h, + Cache::Priority::HIGH)); } ASSERT_GT(cache->GetUsage(), cache->GetCapacity()); expected = {}; @@ -1548,7 +1548,7 @@ void DummyFillCache(Cache& cache, size_t entry_size, size_t charge = std::min(entry_size, capacity - my_usage); Cache::Handle* handle; Status st = cache.Insert(ck.WithOffset(my_usage).AsSlice(), fake_value, - charge, /*deleter*/ nullptr, &handle); + &kNoopCacheItemHelper, charge, &handle); ASSERT_OK(st); handles.emplace_back(&cache, handle); my_usage += charge; diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index 85cd5c04ea..cbc55c9720 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -1848,8 +1848,8 @@ TEST_F(DBPropertiesTest, BlobCacheProperties) { // Insert unpinned blob to the cache and check size. constexpr size_t kSize1 = 70; - ASSERT_OK(blob_cache->Insert("blob1", nullptr /*value*/, kSize1, - nullptr /*deleter*/)); + ASSERT_OK(blob_cache->Insert("blob1", nullptr /*value*/, + &kNoopCacheItemHelper, kSize1)); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); ASSERT_EQ(kCapacity, value); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value)); @@ -1861,8 +1861,8 @@ TEST_F(DBPropertiesTest, BlobCacheProperties) { // Insert pinned blob to the cache and check size. constexpr size_t kSize2 = 60; Cache::Handle* blob2 = nullptr; - ASSERT_OK(blob_cache->Insert("blob2", nullptr /*value*/, kSize2, - nullptr /*deleter*/, &blob2)); + ASSERT_OK(blob_cache->Insert("blob2", nullptr /*value*/, + &kNoopCacheItemHelper, kSize2, &blob2)); ASSERT_NE(nullptr, blob2); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); ASSERT_EQ(kCapacity, value); @@ -1876,8 +1876,8 @@ TEST_F(DBPropertiesTest, BlobCacheProperties) { // Insert another pinned blob to make the cache over-sized. constexpr size_t kSize3 = 80; Cache::Handle* blob3 = nullptr; - ASSERT_OK(blob_cache->Insert("blob3", nullptr /*value*/, kSize3, - nullptr /*deleter*/, &blob3)); + ASSERT_OK(blob_cache->Insert("blob3", nullptr /*value*/, + &kNoopCacheItemHelper, kSize3, &blob3)); ASSERT_NE(nullptr, blob3); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); ASSERT_EQ(kCapacity, value); @@ -1956,8 +1956,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) { // Insert unpinned item to the cache and check size. constexpr size_t kSize1 = 50; - ASSERT_OK(block_cache->Insert("item1", nullptr /*value*/, kSize1, - nullptr /*deleter*/)); + ASSERT_OK(block_cache->Insert("item1", nullptr /*value*/, + &kNoopCacheItemHelper, kSize1)); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); ASSERT_EQ(kCapacity, value); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); @@ -1969,8 +1969,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) { // Insert pinned item to the cache and check size. constexpr size_t kSize2 = 30; Cache::Handle* item2 = nullptr; - ASSERT_OK(block_cache->Insert("item2", nullptr /*value*/, kSize2, - nullptr /*deleter*/, &item2)); + ASSERT_OK(block_cache->Insert("item2", nullptr /*value*/, + &kNoopCacheItemHelper, kSize2, &item2)); ASSERT_NE(nullptr, item2); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); ASSERT_EQ(kCapacity, value); @@ -1983,8 +1983,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) { // Insert another pinned item to make the cache over-sized. constexpr size_t kSize3 = 80; Cache::Handle* item3 = nullptr; - ASSERT_OK(block_cache->Insert("item3", nullptr /*value*/, kSize3, - nullptr /*deleter*/, &item3)); + ASSERT_OK(block_cache->Insert("item3", nullptr /*value*/, + &kNoopCacheItemHelper, kSize3, &item3)); ASSERT_NE(nullptr, item2); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); ASSERT_EQ(kCapacity, value); diff --git a/db/db_test_util.cc b/db/db_test_util.cc index d53bca51a5..2d3b7207c8 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -1723,12 +1723,13 @@ TargetCacheChargeTrackingCache::TargetCacheChargeTrackingCache( cache_charge_increments_sum_(0) {} template -Status TargetCacheChargeTrackingCache::Insert( - const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), Handle** handle, - Priority priority) { - Status s = target_->Insert(key, value, charge, deleter, handle, priority); - if (deleter == kNoopDeleter) { +Status TargetCacheChargeTrackingCache::Insert(const Slice& key, + ObjectPtr value, + const CacheItemHelper* helper, + size_t charge, Handle** handle, + Priority priority) { + Status s = target_->Insert(key, value, helper, charge, handle, priority); + if (helper == kCrmHelper) { if (last_peak_tracked_) { cache_charge_peak_ = 0; cache_charge_increment_ = 0; @@ -1747,8 +1748,8 @@ Status TargetCacheChargeTrackingCache::Insert( template bool TargetCacheChargeTrackingCache::Release(Handle* handle, bool erase_if_last_ref) { - auto deleter = GetDeleter(handle); - if (deleter == kNoopDeleter) { + auto helper = GetCacheItemHelper(handle); + if (helper == kCrmHelper) { if (!last_peak_tracked_) { cache_charge_peaks_.push_back(cache_charge_peak_); cache_charge_increments_sum_ += cache_charge_increment_; @@ -1761,8 +1762,8 @@ bool TargetCacheChargeTrackingCache::Release(Handle* handle, } template -const Cache::DeleterFn TargetCacheChargeTrackingCache::kNoopDeleter = - CacheReservationManagerImpl::TEST_GetNoopDeleterForRole(); +const Cache::CacheItemHelper* TargetCacheChargeTrackingCache::kCrmHelper = + CacheReservationManagerImpl::TEST_GetCacheItemHelperForRole(); template class TargetCacheChargeTrackingCache< CacheEntryRole::kFilterConstruction>; diff --git a/db/db_test_util.h b/db/db_test_util.h index c6f740c92c..c174304229 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -903,17 +903,18 @@ class CacheWrapper : public Cache { const char* Name() const override { return target_->Name(); } - using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, Handle** handle = nullptr, Priority priority = Priority::LOW) override { - return target_->Insert(key, value, charge, deleter, handle, priority); + return target_->Insert(key, value, helper, charge, handle, priority); } - using Cache::Lookup; - Handle* Lookup(const Slice& key, Statistics* stats = nullptr) override { - return target_->Lookup(key, stats); + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr) override { + return target_->Lookup(key, helper, create_context, priority, wait, stats); } bool Ref(Handle* handle) override { return target_->Ref(handle); } @@ -923,7 +924,7 @@ class CacheWrapper : public Cache { return target_->Release(handle, erase_if_last_ref); } - void* Value(Handle* handle) override { return target_->Value(handle); } + ObjectPtr Value(Handle* handle) override { return target_->Value(handle); } void Erase(const Slice& key) override { target_->Erase(key); } uint64_t NewId() override { return target_->NewId(); } @@ -952,18 +953,13 @@ class CacheWrapper : public Cache { return target_->GetCharge(handle); } - DeleterFn GetDeleter(Handle* handle) const override { - return target_->GetDeleter(handle); - } - - void ApplyToAllCacheEntries(void (*callback)(void*, size_t), - bool thread_safe) override { - target_->ApplyToAllCacheEntries(callback, thread_safe); + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override { + return target_->GetCacheItemHelper(handle); } void ApplyToAllEntries( - const std::function& callback, + const std::function& callback, const ApplyToAllEntriesOptions& opts) override { target_->ApplyToAllEntries(callback, opts); } @@ -991,9 +987,8 @@ class TargetCacheChargeTrackingCache : public CacheWrapper { public: explicit TargetCacheChargeTrackingCache(std::shared_ptr target); - using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, Handle** handle = nullptr, Priority priority = Priority::LOW) override; @@ -1009,7 +1004,7 @@ class TargetCacheChargeTrackingCache : public CacheWrapper { } private: - static const Cache::DeleterFn kNoopDeleter; + static const Cache::CacheItemHelper* kCrmHelper; std::size_t cur_cache_charge_; std::size_t cache_charge_peak_; diff --git a/db/internal_stats.cc b/db/internal_stats.cc index ac5b81f3e4..bc7f315d95 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -659,17 +659,18 @@ void InternalStats::CollectCacheEntryStats(bool foreground) { min_interval_factor); } -std::function +std::function Blah() { + static int x = 42; + return [&]() { ++x; }; +} + +std::function InternalStats::CacheEntryRoleStats::GetEntryCallback() { - return [&](const Slice& /*key*/, void* /*value*/, size_t charge, - Cache::DeleterFn deleter) { - auto e = role_map_.find(deleter); - size_t role_idx; - if (e == role_map_.end()) { - role_idx = static_cast(CacheEntryRole::kMisc); - } else { - role_idx = static_cast(e->second); - } + return [&](const Slice& /*key*/, Cache::ObjectPtr /*value*/, size_t charge, + const Cache::CacheItemHelper* helper) -> void { + size_t role_idx = + static_cast(helper ? helper->role : CacheEntryRole::kMisc); entry_counts[role_idx]++; total_charges[role_idx] += charge; }; @@ -680,7 +681,6 @@ void InternalStats::CacheEntryRoleStats::BeginCollection( Clear(); last_start_time_micros_ = start_time_micros; ++collection_count; - role_map_ = CopyCacheDeleterRoleMap(); std::ostringstream str; str << cache->Name() << "@" << static_cast(cache) << "#" << port::GetProcessID(); diff --git a/db/internal_stats.h b/db/internal_stats.h index b0cd5899b8..a833240637 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -472,7 +472,8 @@ class InternalStats { } void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros); - std::function + std::function GetEntryCallback(); void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros); void SkippedCollection(); @@ -482,7 +483,6 @@ class InternalStats { SystemClock* clock) const; private: - UnorderedMap role_map_; uint64_t GetLastDurationMicros() const; }; diff --git a/db/table_cache.cc b/db/table_cache.cc index c44c4bb84d..a5fa5fbe3c 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -31,16 +31,6 @@ #include "util/coding.h" #include "util/stop_watch.h" -namespace ROCKSDB_NAMESPACE { -namespace { -template -static void DeleteEntry(const Slice& /*key*/, void* value) { - T* typed_value = reinterpret_cast(value); - delete typed_value; -} -} // anonymous namespace -} // namespace ROCKSDB_NAMESPACE - // Generate the regular and coroutine versions of some methods by // including table_cache_sync_and_async.h twice // Macros in the header will expand differently based on whether @@ -58,12 +48,6 @@ namespace ROCKSDB_NAMESPACE { namespace { -static void UnrefEntry(void* arg1, void* arg2) { - Cache* cache = reinterpret_cast(arg1); - Cache::Handle* h = reinterpret_cast(arg2); - cache->Release(h); -} - static Slice GetSliceForFileNumber(const uint64_t* file_number) { return Slice(reinterpret_cast(file_number), sizeof(*file_number)); @@ -105,14 +89,6 @@ TableCache::TableCache(const ImmutableOptions& ioptions, TableCache::~TableCache() {} -TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) { - return reinterpret_cast(cache_->Value(handle)); -} - -void TableCache::ReleaseHandle(Cache::Handle* handle) { - cache_->Release(handle); -} - Status TableCache::GetTableReader( const ReadOptions& ro, const FileOptions& file_options, const InternalKeyComparator& internal_comparator, @@ -178,17 +154,10 @@ Status TableCache::GetTableReader( return s; } -void TableCache::EraseHandle(const FileDescriptor& fd, Cache::Handle* handle) { - ReleaseHandle(handle); - uint64_t number = fd.GetNumber(); - Slice key = GetSliceForFileNumber(&number); - cache_->Erase(key); -} - Status TableCache::FindTable( const ReadOptions& ro, const FileOptions& file_options, const InternalKeyComparator& internal_comparator, - const FileMetaData& file_meta, Cache::Handle** handle, + const FileMetaData& file_meta, TypedHandle** handle, const std::shared_ptr& prefix_extractor, const bool no_io, bool record_read_stats, HistogramImpl* file_read_hist, bool skip_filters, int level, bool prefetch_index_and_filter_in_cache, @@ -196,7 +165,7 @@ Status TableCache::FindTable( PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock); uint64_t number = file_meta.fd.GetNumber(); Slice key = GetSliceForFileNumber(&number); - *handle = cache_->Lookup(key); + *handle = cache_.Lookup(key); TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0", const_cast(&no_io)); @@ -206,7 +175,7 @@ Status TableCache::FindTable( } MutexLock load_lock(loader_mutex_.get(key)); // We check the cache again under loading mutex - *handle = cache_->Lookup(key); + *handle = cache_.Lookup(key); if (*handle != nullptr) { return Status::OK(); } @@ -224,8 +193,7 @@ Status TableCache::FindTable( // We do not cache error results so that if the error is transient, // or somebody repairs the file, we recover automatically. } else { - s = cache_->Insert(key, table_reader.get(), 1, &DeleteEntry, - handle); + s = cache_.Insert(key, table_reader.get(), 1, handle); if (s.ok()) { // Release ownership of table reader. table_reader.release(); @@ -251,7 +219,7 @@ InternalIterator* TableCache::NewIterator( Status s; TableReader* table_reader = nullptr; - Cache::Handle* handle = nullptr; + TypedHandle* handle = nullptr; if (table_reader_ptr != nullptr) { *table_reader_ptr = nullptr; } @@ -266,7 +234,7 @@ InternalIterator* TableCache::NewIterator( level, true /* prefetch_index_and_filter_in_cache */, max_file_size_for_l0_meta_pin, file_meta.temperature); if (s.ok()) { - table_reader = GetTableReaderFromHandle(handle); + table_reader = cache_.Value(handle); } } InternalIterator* result = nullptr; @@ -280,7 +248,7 @@ InternalIterator* TableCache::NewIterator( file_options.compaction_readahead_size, allow_unprepared_value); } if (handle != nullptr) { - result->RegisterCleanup(&UnrefEntry, cache_, handle); + cache_.RegisterReleaseAsCleanup(handle, *result); handle = nullptr; // prevent from releasing below } @@ -330,7 +298,7 @@ InternalIterator* TableCache::NewIterator( } if (handle != nullptr) { - ReleaseHandle(handle); + cache_.Release(handle); } if (!s.ok()) { assert(result == nullptr); @@ -348,12 +316,12 @@ Status TableCache::GetRangeTombstoneIterator( const FileDescriptor& fd = file_meta.fd; Status s; TableReader* t = fd.table_reader; - Cache::Handle* handle = nullptr; + TypedHandle* handle = nullptr; if (t == nullptr) { s = FindTable(options, file_options_, internal_comparator, file_meta, &handle); if (s.ok()) { - t = GetTableReaderFromHandle(handle); + t = cache_.Value(handle); } } if (s.ok()) { @@ -362,9 +330,9 @@ Status TableCache::GetRangeTombstoneIterator( } if (handle) { if (*out_iter) { - (*out_iter)->RegisterCleanup(&UnrefEntry, cache_, handle); + cache_.RegisterReleaseAsCleanup(handle, **out_iter); } else { - ReleaseHandle(handle); + cache_.Release(handle); } } return s; @@ -411,16 +379,10 @@ bool TableCache::GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, bool found = false; row_cache_key.TrimAppend(prefix_size, user_key.data(), user_key.size()); - if (auto row_handle = - ioptions_.row_cache->Lookup(row_cache_key.GetUserKey())) { + RowCacheInterface row_cache{ioptions_.row_cache.get()}; + if (auto row_handle = row_cache.Lookup(row_cache_key.GetUserKey())) { // Cleanable routine to release the cache entry Cleanable value_pinner; - auto release_cache_entry_func = [](void* cache_to_clean, - void* cache_handle) { - ((Cache*)cache_to_clean)->Release((Cache::Handle*)cache_handle); - }; - auto found_row_cache_entry = - static_cast(ioptions_.row_cache->Value(row_handle)); // If it comes here value is located on the cache. // found_row_cache_entry points to the value on cache, // and value_pinner has cleanup procedure for the cached entry. @@ -429,9 +391,8 @@ bool TableCache::GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, // cleanup routine under value_pinner will be delegated to // get_context.pinnable_slice_. Cache entry is released when // get_context.pinnable_slice_ is reset. - value_pinner.RegisterCleanup(release_cache_entry_func, - ioptions_.row_cache.get(), row_handle); - replayGetContextLog(*found_row_cache_entry, user_key, get_context, + row_cache.RegisterReleaseAsCleanup(row_handle, value_pinner); + replayGetContextLog(*row_cache.Value(row_handle), user_key, get_context, &value_pinner); RecordTick(ioptions_.stats, ROW_CACHE_HIT); found = true; @@ -470,7 +431,7 @@ Status TableCache::Get( #endif // ROCKSDB_LITE Status s; TableReader* t = fd.table_reader; - Cache::Handle* handle = nullptr; + TypedHandle* handle = nullptr; if (!done) { assert(s.ok()); if (t == nullptr) { @@ -481,7 +442,7 @@ Status TableCache::Get( level, true /* prefetch_index_and_filter_in_cache */, max_file_size_for_l0_meta_pin, file_meta.temperature); if (s.ok()) { - t = GetTableReaderFromHandle(handle); + t = cache_.Value(handle); } } SequenceNumber* max_covering_tombstone_seq = @@ -517,18 +478,17 @@ Status TableCache::Get( #ifndef ROCKSDB_LITE // Put the replay log in row cache only if something was found. if (!done && s.ok() && row_cache_entry && !row_cache_entry->empty()) { + RowCacheInterface row_cache{ioptions_.row_cache.get()}; size_t charge = row_cache_entry->capacity() + sizeof(std::string); - void* row_ptr = new std::string(std::move(*row_cache_entry)); + auto row_ptr = new std::string(std::move(*row_cache_entry)); // If row cache is full, it's OK to continue. - ioptions_.row_cache - ->Insert(row_cache_key.GetUserKey(), row_ptr, charge, - &DeleteEntry) + row_cache.Insert(row_cache_key.GetUserKey(), row_ptr, charge) .PermitUncheckedError(); } #endif // ROCKSDB_LITE if (handle != nullptr) { - ReleaseHandle(handle); + cache_.Release(handle); } return s; } @@ -561,7 +521,7 @@ Status TableCache::MultiGetFilter( const FileMetaData& file_meta, const std::shared_ptr& prefix_extractor, HistogramImpl* file_read_hist, int level, - MultiGetContext::Range* mget_range, Cache::Handle** table_handle) { + MultiGetContext::Range* mget_range, TypedHandle** table_handle) { auto& fd = file_meta.fd; #ifndef ROCKSDB_LITE IterKey row_cache_key; @@ -577,7 +537,7 @@ Status TableCache::MultiGetFilter( #endif // ROCKSDB_LITE Status s; TableReader* t = fd.table_reader; - Cache::Handle* handle = nullptr; + TypedHandle* handle = nullptr; MultiGetContext::Range tombstone_range(*mget_range, mget_range->begin(), mget_range->end()); if (t == nullptr) { @@ -588,7 +548,7 @@ Status TableCache::MultiGetFilter( level, true /* prefetch_index_and_filter_in_cache */, /*max_file_size_for_l0_meta_pin=*/0, file_meta.temperature); if (s.ok()) { - t = GetTableReaderFromHandle(handle); + t = cache_.Value(handle); } *table_handle = handle; } @@ -602,7 +562,7 @@ Status TableCache::MultiGetFilter( UpdateRangeTombstoneSeqnums(options, t, tombstone_range); } if (mget_range->empty() && handle) { - ReleaseHandle(handle); + cache_.Release(handle); *table_handle = nullptr; } @@ -623,16 +583,16 @@ Status TableCache::GetTableProperties( return Status::OK(); } - Cache::Handle* table_handle = nullptr; + TypedHandle* table_handle = nullptr; Status s = FindTable(ReadOptions(), file_options, internal_comparator, file_meta, &table_handle, prefix_extractor, no_io); if (!s.ok()) { return s; } assert(table_handle); - auto table = GetTableReaderFromHandle(table_handle); + auto table = cache_.Value(table_handle); *properties = table->GetTableProperties(); - ReleaseHandle(table_handle); + cache_.Release(table_handle); return s; } @@ -641,18 +601,18 @@ Status TableCache::ApproximateKeyAnchors( const FileMetaData& file_meta, std::vector& anchors) { Status s; TableReader* t = file_meta.fd.table_reader; - Cache::Handle* handle = nullptr; + TypedHandle* handle = nullptr; if (t == nullptr) { s = FindTable(ro, file_options_, internal_comparator, file_meta, &handle); if (s.ok()) { - t = GetTableReaderFromHandle(handle); + t = cache_.Value(handle); } } if (s.ok() && t != nullptr) { s = t->ApproximateKeyAnchors(ro, anchors); } if (handle != nullptr) { - ReleaseHandle(handle); + cache_.Release(handle); } return s; } @@ -668,29 +628,19 @@ size_t TableCache::GetMemoryUsageByTableReader( return table_reader->ApproximateMemoryUsage(); } - Cache::Handle* table_handle = nullptr; + TypedHandle* table_handle = nullptr; Status s = FindTable(ReadOptions(), file_options, internal_comparator, file_meta, &table_handle, prefix_extractor, true); if (!s.ok()) { return 0; } assert(table_handle); - auto table = GetTableReaderFromHandle(table_handle); + auto table = cache_.Value(table_handle); auto ret = table->ApproximateMemoryUsage(); - ReleaseHandle(table_handle); + cache_.Release(table_handle); return ret; } -bool TableCache::HasEntry(Cache* cache, uint64_t file_number) { - Cache::Handle* handle = cache->Lookup(GetSliceForFileNumber(&file_number)); - if (handle) { - cache->Release(handle); - return true; - } else { - return false; - } -} - void TableCache::Evict(Cache* cache, uint64_t file_number) { cache->Erase(GetSliceForFileNumber(&file_number)); } @@ -701,7 +651,7 @@ uint64_t TableCache::ApproximateOffsetOf( const std::shared_ptr& prefix_extractor) { uint64_t result = 0; TableReader* table_reader = file_meta.fd.table_reader; - Cache::Handle* table_handle = nullptr; + TypedHandle* table_handle = nullptr; if (table_reader == nullptr) { const bool for_compaction = (caller == TableReaderCaller::kCompaction); Status s = @@ -709,7 +659,7 @@ uint64_t TableCache::ApproximateOffsetOf( &table_handle, prefix_extractor, false /* no_io */, !for_compaction /* record_read_stats */); if (s.ok()) { - table_reader = GetTableReaderFromHandle(table_handle); + table_reader = cache_.Value(table_handle); } } @@ -717,7 +667,7 @@ uint64_t TableCache::ApproximateOffsetOf( result = table_reader->ApproximateOffsetOf(key, caller); } if (table_handle != nullptr) { - ReleaseHandle(table_handle); + cache_.Release(table_handle); } return result; @@ -729,7 +679,7 @@ uint64_t TableCache::ApproximateSize( const std::shared_ptr& prefix_extractor) { uint64_t result = 0; TableReader* table_reader = file_meta.fd.table_reader; - Cache::Handle* table_handle = nullptr; + TypedHandle* table_handle = nullptr; if (table_reader == nullptr) { const bool for_compaction = (caller == TableReaderCaller::kCompaction); Status s = @@ -737,7 +687,7 @@ uint64_t TableCache::ApproximateSize( &table_handle, prefix_extractor, false /* no_io */, !for_compaction /* record_read_stats */); if (s.ok()) { - table_reader = GetTableReaderFromHandle(table_handle); + table_reader = cache_.Value(table_handle); } } @@ -745,7 +695,7 @@ uint64_t TableCache::ApproximateSize( result = table_reader->ApproximateSize(start, end, caller); } if (table_handle != nullptr) { - ReleaseHandle(table_handle); + cache_.Release(table_handle); } return result; diff --git a/db/table_cache.h b/db/table_cache.h index 2e50f2c779..66282bf41f 100644 --- a/db/table_cache.h +++ b/db/table_cache.h @@ -14,6 +14,7 @@ #include #include +#include "cache/typed_cache.h" #include "db/dbformat.h" #include "db/range_del_aggregator.h" #include "options/cf_options.h" @@ -56,6 +57,16 @@ class TableCache { const std::string& db_session_id); ~TableCache(); + // Cache interface for table cache + using CacheInterface = + BasicTypedCacheInterface; + using TypedHandle = CacheInterface::TypedHandle; + + // Cache interface for row cache + using RowCacheInterface = + BasicTypedCacheInterface; + using RowHandle = RowCacheInterface::TypedHandle; + // Return an iterator for the specified file number (the corresponding // file length must be exactly "file_size" bytes). If "table_reader_ptr" // is non-nullptr, also sets "*table_reader_ptr" to point to the Table object @@ -124,7 +135,7 @@ class TableCache { const FileMetaData& file_meta, const std::shared_ptr& prefix_extractor, HistogramImpl* file_read_hist, int level, - MultiGetContext::Range* mget_range, Cache::Handle** table_handle); + MultiGetContext::Range* mget_range, TypedHandle** table_handle); // If a seek to internal key "k" in specified file finds an entry, // call get_context->SaveValue() repeatedly until @@ -142,25 +153,18 @@ class TableCache { const std::shared_ptr& prefix_extractor = nullptr, HistogramImpl* file_read_hist = nullptr, bool skip_filters = false, bool skip_range_deletions = false, int level = -1, - Cache::Handle* table_handle = nullptr); + TypedHandle* table_handle = nullptr); // Evict any entry for the specified file number static void Evict(Cache* cache, uint64_t file_number); - // Query whether specified file number is currently in cache - static bool HasEntry(Cache* cache, uint64_t file_number); - - // Clean table handle and erase it from the table cache - // Used in DB close, or the file is not live anymore. - void EraseHandle(const FileDescriptor& fd, Cache::Handle* handle); - // Find table reader // @param skip_filters Disables loading/accessing the filter block // @param level == -1 means not specified Status FindTable( const ReadOptions& ro, const FileOptions& toptions, const InternalKeyComparator& internal_comparator, - const FileMetaData& file_meta, Cache::Handle**, + const FileMetaData& file_meta, TypedHandle**, const std::shared_ptr& prefix_extractor = nullptr, const bool no_io = false, bool record_read_stats = true, HistogramImpl* file_read_hist = nullptr, bool skip_filters = false, @@ -168,9 +172,6 @@ class TableCache { size_t max_file_size_for_l0_meta_pin = 0, Temperature file_temperature = Temperature::kUnknown); - // Get TableReader from a cache handle. - TableReader* GetTableReaderFromHandle(Cache::Handle* handle); - // Get the table properties of a given table. // @no_io: indicates if we should load table to the cache if it is not present // in table cache yet. @@ -212,10 +213,7 @@ class TableCache { const InternalKeyComparator& internal_comparator, const std::shared_ptr& prefix_extractor = nullptr); - // Release the handle from a cache - void ReleaseHandle(Cache::Handle* handle); - - Cache* get_cache() const { return cache_; } + CacheInterface& get_cache() { return cache_; } // Capacity of the backing Cache that indicates infinite TableCache capacity. // For example when max_open_files is -1 we set the backing Cache to this. @@ -224,7 +222,7 @@ class TableCache { // The tables opened with this TableCache will be immortal, i.e., their // lifetime is as long as that of the DB. void SetTablesAreImmortal() { - if (cache_->GetCapacity() >= kInfiniteCapacity) { + if (cache_.get()->GetCapacity() >= kInfiniteCapacity) { immortal_tables_ = true; } } @@ -263,7 +261,7 @@ class TableCache { const ImmutableOptions& ioptions_; const FileOptions& file_options_; - Cache* const cache_; + CacheInterface cache_; std::string row_cache_id_; bool immortal_tables_; BlockCacheTracer* const block_cache_tracer_; diff --git a/db/table_cache_sync_and_async.h b/db/table_cache_sync_and_async.h index e72abdd45d..9043ec8363 100644 --- a/db/table_cache_sync_and_async.h +++ b/db/table_cache_sync_and_async.h @@ -19,15 +19,14 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) const FileMetaData& file_meta, const MultiGetContext::Range* mget_range, const std::shared_ptr& prefix_extractor, HistogramImpl* file_read_hist, bool skip_filters, bool skip_range_deletions, - int level, Cache::Handle* table_handle) { + int level, TypedHandle* handle) { auto& fd = file_meta.fd; Status s; TableReader* t = fd.table_reader; - Cache::Handle* handle = table_handle; MultiGetRange table_range(*mget_range, mget_range->begin(), mget_range->end()); if (handle != nullptr && t == nullptr) { - t = GetTableReaderFromHandle(handle); + t = cache_.Value(handle); } #ifndef ROCKSDB_LITE autovector row_cache_entries; @@ -75,7 +74,7 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) 0 /*max_file_size_for_l0_meta_pin*/, file_meta.temperature); TEST_SYNC_POINT_CALLBACK("TableCache::MultiGet:FindTable", &s); if (s.ok()) { - t = GetTableReaderFromHandle(handle); + t = cache_.Value(handle); assert(t); } } @@ -100,6 +99,7 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) #ifndef ROCKSDB_LITE if (lookup_row_cache) { size_t row_idx = 0; + RowCacheInterface row_cache{ioptions_.row_cache.get()}; for (auto miter = table_range.begin(); miter != table_range.end(); ++miter) { @@ -115,11 +115,9 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) // Put the replay log in row cache only if something was found. if (s.ok() && !row_cache_entry.empty()) { size_t charge = row_cache_entry.capacity() + sizeof(std::string); - void* row_ptr = new std::string(std::move(row_cache_entry)); + auto row_ptr = new std::string(std::move(row_cache_entry)); // If row cache is full, it's OK. - ioptions_.row_cache - ->Insert(row_cache_key.GetUserKey(), row_ptr, charge, - &DeleteEntry) + row_cache.Insert(row_cache_key.GetUserKey(), row_ptr, charge) .PermitUncheckedError(); } } @@ -127,7 +125,7 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) #endif // ROCKSDB_LITE if (handle != nullptr) { - ReleaseHandle(handle); + cache_.Release(handle); } CO_RETURN s; } diff --git a/db/version_builder.cc b/db/version_builder.cc index bff90b242f..4f0e3a8413 100644 --- a/db/version_builder.cc +++ b/db/version_builder.cc @@ -294,7 +294,9 @@ class VersionBuilder::Rep { if (f->refs <= 0) { if (f->table_reader_handle) { assert(table_cache_ != nullptr); - table_cache_->ReleaseHandle(f->table_reader_handle); + // NOTE: have to release in raw cache interface to avoid using a + // TypedHandle for FileMetaData::table_reader_handle + table_cache_->get_cache().get()->Release(f->table_reader_handle); f->table_reader_handle = nullptr; } @@ -1258,7 +1260,8 @@ class VersionBuilder::Rep { size_t max_file_size_for_l0_meta_pin) { assert(table_cache_ != nullptr); - size_t table_cache_capacity = table_cache_->get_cache()->GetCapacity(); + size_t table_cache_capacity = + table_cache_->get_cache().get()->GetCapacity(); bool always_load = (table_cache_capacity == TableCache::kInfiniteCapacity); size_t max_load = std::numeric_limits::max(); @@ -1280,7 +1283,7 @@ class VersionBuilder::Rep { load_limit = table_cache_capacity / 4; } - size_t table_cache_usage = table_cache_->get_cache()->GetUsage(); + size_t table_cache_usage = table_cache_->get_cache().get()->GetUsage(); if (table_cache_usage >= load_limit) { // TODO (yanqin) find a suitable status code. return Status::OK(); @@ -1319,18 +1322,18 @@ class VersionBuilder::Rep { auto* file_meta = files_meta[file_idx].first; int level = files_meta[file_idx].second; + TableCache::TypedHandle* handle = nullptr; statuses[file_idx] = table_cache_->FindTable( ReadOptions(), file_options_, - *(base_vstorage_->InternalComparator()), *file_meta, - &file_meta->table_reader_handle, prefix_extractor, false /*no_io */, - true /* record_read_stats */, + *(base_vstorage_->InternalComparator()), *file_meta, &handle, + prefix_extractor, false /*no_io */, true /* record_read_stats */, internal_stats->GetFileReadHist(level), false, level, prefetch_index_and_filter_in_cache, max_file_size_for_l0_meta_pin, file_meta->temperature); - if (file_meta->table_reader_handle != nullptr) { + if (handle != nullptr) { + file_meta->table_reader_handle = handle; // Load table_reader - file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle( - file_meta->table_reader_handle); + file_meta->fd.table_reader = table_cache_->get_cache().Value(handle); } } }); diff --git a/db/version_set.cc b/db/version_set.cc index be1db7ba3e..886d8dc3e7 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -2511,7 +2511,7 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range, std::vector> mget_tasks; while (f != nullptr) { MultiGetRange file_range = fp.CurrentFileRange(); - Cache::Handle* table_handle = nullptr; + TableCache::TypedHandle* table_handle = nullptr; bool skip_filters = IsFilterSkipped(static_cast(fp.GetHitFileLevel()), fp.IsHitFileLastInLevel()); @@ -2693,7 +2693,7 @@ Status Version::ProcessBatch( } while (f) { MultiGetRange file_range = fp.CurrentFileRange(); - Cache::Handle* table_handle = nullptr; + TableCache::TypedHandle* table_handle = nullptr; bool skip_filters = IsFilterSkipped(static_cast(fp.GetHitFileLevel()), fp.IsHitFileLastInLevel()); bool skip_range_deletions = false; @@ -6879,16 +6879,16 @@ Status VersionSet::VerifyFileMetadata(ColumnFamilyData* cfd, InternalStats* internal_stats = cfd->internal_stats(); + TableCache::TypedHandle* handle = nullptr; FileMetaData meta_copy = meta; status = table_cache->FindTable( - ReadOptions(), file_opts, *icmp, meta_copy, - &(meta_copy.table_reader_handle), pe, + ReadOptions(), file_opts, *icmp, meta_copy, &handle, pe, /*no_io=*/false, /*record_read_stats=*/true, internal_stats->GetFileReadHist(level), false, level, /*prefetch_index_and_filter_in_cache*/ false, max_sz_for_l0_meta_pin, meta_copy.temperature); - if (meta_copy.table_reader_handle) { - table_cache->ReleaseHandle(meta_copy.table_reader_handle); + if (handle) { + table_cache->get_cache().Release(handle); } } return status; diff --git a/db/version_set.h b/db/version_set.h index b92546ed62..a5893e4a16 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -1022,7 +1022,7 @@ class Version { int hit_file_level, bool skip_filters, bool skip_range_deletions, FdWithKeyRange* f, std::unordered_map& blob_ctxs, - Cache::Handle* table_handle, uint64_t& num_filter_read, + TableCache::TypedHandle* table_handle, uint64_t& num_filter_read, uint64_t& num_index_read, uint64_t& num_sst_read); #ifdef USE_COROUTINES @@ -1431,7 +1431,7 @@ class VersionSet { void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) { assert(table_cache_); - table_cache_->Erase(GetSlice(&blob_file_number)); + table_cache_->Erase(GetSliceForKey(&blob_file_number)); obsolete_blob_files_.emplace_back(blob_file_number, std::move(path)); } diff --git a/db/version_set_sync_and_async.h b/db/version_set_sync_and_async.h index 7555859902..51f58cdad0 100644 --- a/db/version_set_sync_and_async.h +++ b/db/version_set_sync_and_async.h @@ -16,7 +16,7 @@ DEFINE_SYNC_AND_ASYNC(Status, Version::MultiGetFromSST) (const ReadOptions& read_options, MultiGetRange file_range, int hit_file_level, bool skip_filters, bool skip_range_deletions, FdWithKeyRange* f, std::unordered_map& blob_ctxs, - Cache::Handle* table_handle, uint64_t& num_filter_read, + TableCache::TypedHandle* table_handle, uint64_t& num_filter_read, uint64_t& num_index_read, uint64_t& num_sst_read) { bool timer_enabled = GetPerfLevel() >= PerfLevel::kEnableTimeExceptForMutex && get_perf_context()->per_level_perf_context_enabled; diff --git a/include/rocksdb/cache.h b/include/rocksdb/cache.h index 575d276b58..584e119bc8 100644 --- a/include/rocksdb/cache.h +++ b/include/rocksdb/cache.h @@ -7,18 +7,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // -// A Cache is an interface that maps keys to values. It has internal -// synchronization and may be safely accessed concurrently from -// multiple threads. It may automatically evict entries to make room -// for new entries. Values have a specified charge against the cache -// capacity. For example, a cache where the values are variable -// length strings, may use the length of the string as the charge for -// the string. -// -// A builtin cache implementation with a least-recently-used eviction -// policy is provided. Clients may use their own implementations if -// they want something more sophisticated (like scan-resistance, a -// custom eviction policy, variable cache sizing, etc.) +// Various APIs for creating and customizing read caches in RocksDB. #pragma once @@ -363,11 +352,33 @@ extern std::shared_ptr NewClockCache( CacheMetadataChargePolicy metadata_charge_policy = kDefaultCacheMetadataChargePolicy); +// A Cache maps keys to objects resident in memory, tracks reference counts +// on those key-object entries, and is able to remove unreferenced entries +// whenever it wants. All operations are fully thread safe except as noted. +// Inserted entries have a specified "charge" which is some quantity in +// unspecified units, typically bytes of memory used. A Cache will typically +// have a finite capacity in units of charge, and evict entries as needed +// to stay at or below that capacity. +// +// NOTE: This API is for expert use only and is more intended for providing +// custom implementations than for calling into. It is subject to change +// as RocksDB evolves, especially the RocksDB block cache. +// +// INTERNAL: See typed_cache.h for convenient wrappers on top of this API. class Cache { - public: // opaque types + public: // types hidden from API client // Opaque handle to an entry stored in the cache. struct Handle {}; + public: // types hidden from Cache implementation + // Pointer to cached object of unspecified type. (This type alias is + // provided for clarity, not really for type checking.) + using ObjectPtr = void*; + + // Opaque object providing context (settings, etc.) to create objects + // for primary cache from saved (serialized) secondary cache entries. + struct CreateContext {}; + public: // type defs // Depending on implementation, cache entries with higher priority levels // could be less likely to get evicted than entries with lower priority @@ -400,48 +411,84 @@ class Cache { // so anything required for these operations should be contained in the // object itself. // - // The SizeCallback takes a void* pointer to the object and returns the size + // The SizeCallback takes a pointer to the object and returns the size // of the persistable data. It can be used by the secondary cache to allocate // memory if needed. // // RocksDB callbacks are NOT exception-safe. A callback completing with an // exception can lead to undefined behavior in RocksDB, including data loss, // unreported corruption, deadlocks, and more. - using SizeCallback = size_t (*)(void* obj); + using SizeCallback = size_t (*)(ObjectPtr obj); - // The SaveToCallback takes a void* object pointer and saves the persistable + // The SaveToCallback takes an object pointer and saves the persistable // data into a buffer. The secondary cache may decide to not store it in a // contiguous buffer, in which case this callback will be called multiple // times with increasing offset - using SaveToCallback = Status (*)(void* from_obj, size_t from_offset, - size_t length, void* out); + using SaveToCallback = Status (*)(ObjectPtr from_obj, size_t from_offset, + size_t length, char* out_buf); - // A function pointer type for custom destruction of an entry's - // value. The Cache is responsible for copying and reclaiming space - // for the key, but values are managed by the caller. - using DeleterFn = void (*)(const Slice& key, void* value); + // A function pointer type for destruction of a cache object. This will + // typically call the destructor for the appropriate type of the object. + // The Cache is responsible for copying and reclaiming space for the key, + // but objects are managed in part using this callback. Generally a DeleterFn + // can be nullptr if the ObjectPtr does not need destruction (e.g. nullptr or + // pointer into static data). + using DeleterFn = void (*)(ObjectPtr obj, MemoryAllocator* allocator); + + // The CreateCallback is takes in a buffer from the NVM cache and constructs + // an object using it. The callback doesn't have ownership of the buffer and + // should copy the contents into its own buffer. The CreateContext* is + // provided by Lookup and may be used to follow DB- or CF-specific settings. + // In case of some error, non-OK is returned and the caller should ignore + // any result in out_obj. (The implementation must clean up after itself.) + using CreateCallback = Status (*)(const Slice& data, CreateContext* context, + MemoryAllocator* allocator, + ObjectPtr* out_obj, size_t* out_charge); // A struct with pointers to helper functions for spilling items from the // cache into the secondary cache. May be extended in the future. An // instance of this struct is expected to outlive the cache. struct CacheItemHelper { + // Function for deleting an object on its removal from the Cache. + // nullptr is only for entries that require no destruction, such as + // "placeholder" cache entries with nullptr object. + DeleterFn del_cb; // (<- Most performance critical) + // Next three are used for persisting values as described above. + // If any is nullptr, then all three should be nullptr and persisting the + // entry to/from secondary cache is not supported. SizeCallback size_cb; SaveToCallback saveto_cb; - DeleterFn del_cb; + CreateCallback create_cb; + // Classification of the entry for monitoring purposes in block cache. + CacheEntryRole role; - CacheItemHelper() : size_cb(nullptr), saveto_cb(nullptr), del_cb(nullptr) {} - CacheItemHelper(SizeCallback _size_cb, SaveToCallback _saveto_cb, - DeleterFn _del_cb) - : size_cb(_size_cb), saveto_cb(_saveto_cb), del_cb(_del_cb) {} + constexpr CacheItemHelper() + : del_cb(nullptr), + size_cb(nullptr), + saveto_cb(nullptr), + create_cb(nullptr), + role(CacheEntryRole::kMisc) {} + + explicit constexpr CacheItemHelper(CacheEntryRole _role, + DeleterFn _del_cb = nullptr, + SizeCallback _size_cb = nullptr, + SaveToCallback _saveto_cb = nullptr, + CreateCallback _create_cb = nullptr) + : del_cb(_del_cb), + size_cb(_size_cb), + saveto_cb(_saveto_cb), + create_cb(_create_cb), + role(_role) { + // Either all three secondary cache callbacks are non-nullptr or + // all three are nullptr + assert((size_cb != nullptr) == (saveto_cb != nullptr)); + assert((size_cb != nullptr) == (create_cb != nullptr)); + } + inline bool IsSecondaryCacheCompatible() const { + return size_cb != nullptr; + } }; - // The CreateCallback is passed by the block cache user to Lookup(). It - // takes in a buffer from the NVM cache and constructs an object using - // it. The callback doesn't have ownership of the buffer and should - // copy the contents into its own buffer. - using CreateCallback = std::function; - public: // ctor/dtor/create Cache(std::shared_ptr allocator = nullptr) : memory_allocator_(std::move(allocator)) {} @@ -471,8 +518,6 @@ class Cache { // The type of the Cache virtual const char* Name() const = 0; - // EXPERIMENTAL SecondaryCache support: - // Some APIs here are experimental and might change in the future. // The Insert and Lookup APIs below are intended to allow cached objects // to be demoted/promoted between the primary block cache and a secondary // cache. The secondary cache could be a non-volatile cache, and will @@ -484,46 +529,27 @@ class Cache { // multiple DBs share the same cache and the set of DBs can change // over time. - // Insert a mapping from key->value into the volatile cache only - // and assign it with the specified charge against the total cache capacity. - // If strict_capacity_limit is true and cache reaches its full capacity, - // return Status::MemoryLimit. - // - // If handle is not nullptr, returns a handle that corresponds to the - // mapping. The caller must call this->Release(handle) when the returned - // mapping is no longer needed. In case of error caller is responsible to - // cleanup the value (i.e. calling "deleter"). - // - // If handle is nullptr, it is as if Release is called immediately after - // insert. In case of error value will be cleanup. - // - // When the inserted entry is no longer needed, the key and - // value will be passed to "deleter" which must delete the value. - // (The Cache is responsible for copying and reclaiming space for - // the key.) - virtual Status Insert(const Slice& key, void* value, size_t charge, - DeleterFn deleter, Handle** handle = nullptr, - Priority priority = Priority::LOW) = 0; - - // EXPERIMENTAL - // Insert a mapping from key->value into the cache and assign it + // Insert a mapping from key->object into the cache and assign it // the specified charge against the total cache capacity. If // strict_capacity_limit is true and cache reaches its full capacity, - // return Status::MemoryLimit. `value` must be non-nullptr for this - // Insert() because Value() == nullptr is reserved for indicating failure - // with secondary-cache-compatible mappings. + // return Status::MemoryLimit. `obj` must be non-nullptr if compatible + // with secondary cache (helper->size_cb != nullptr), because Value() == + // nullptr is reserved for indicating some secondary cache failure cases. + // On success, returns OK and takes ownership of `obj`, eventually deleting + // it with helper->del_cb. On non-OK return, the caller maintains ownership + // of `obj` so will often need to delete it in such cases. // // The helper argument is saved by the cache and will be used when the - // inserted object is evicted or promoted to the secondary cache. It, - // therefore, must outlive the cache. + // inserted object is evicted or considered for promotion to the secondary + // cache. Promotion to secondary cache is only enabled if helper->size_cb + // != nullptr. The helper must outlive the cache. Callers may use + // &kNoopCacheItemHelper as a trivial helper (no deleter for the object, + // no secondary cache). `helper` must not be nullptr (efficiency). // - // If handle is not nullptr, returns a handle that corresponds to the - // mapping. The caller must call this->Release(handle) when the returned - // mapping is no longer needed. In case of error caller is responsible to - // cleanup the value (i.e. calling "deleter"). - // - // If handle is nullptr, it is as if Release is called immediately after - // insert. In case of error value will be cleanup. + // If `handle` is not nullptr and return status is OK, `handle` is set + // to a Handle* for the entry. The caller must call this->Release(handle) + // when the returned entry is no longer needed. If `handle` is nullptr, it is + // as if Release is called immediately after Insert. // // Regardless of whether the item was inserted into the cache, // it will attempt to insert it into the secondary cache if one is @@ -532,42 +558,23 @@ class Cache { // the item is only inserted into the primary cache. It may // defer the insertion to the secondary cache as it sees fit. // - // When the inserted entry is no longer needed, the key and - // value will be passed to "deleter". - virtual Status Insert(const Slice& key, void* value, + // When the inserted entry is no longer needed, it will be destroyed using + // helper->del_cb (if non-nullptr). + virtual Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, size_t charge, Handle** handle = nullptr, - Priority priority = Priority::LOW) { - if (!helper) { - return Status::InvalidArgument(); - } - return Insert(key, value, charge, helper->del_cb, handle, priority); - } + Priority priority = Priority::LOW) = 0; - // If the cache has no mapping for "key", returns nullptr. + // Lookup the key, returning nullptr if not found. If found, returns + // a handle to the mapping that must eventually be passed to Release(). // - // Else return a handle that corresponds to the mapping. The caller - // must call this->Release(handle) when the returned mapping is no - // longer needed. - // If stats is not nullptr, relative tickers could be used inside the - // function. - virtual Handle* Lookup(const Slice& key, Statistics* stats = nullptr) = 0; - - // EXPERIMENTAL - // Lookup the key in the primary and secondary caches (if one is configured). - // The create_cb callback function object will be used to contruct the - // cached object. - // If none of the caches have the mapping for the key, returns nullptr. - // Else, returns a handle that corresponds to the mapping. - // - // This call may promote the object from the secondary cache (if one is - // configured, and has the given key) to the primary cache. - // - // The helper argument should be provided if the caller wants the lookup - // to include the secondary cache (if one is configured) and the object, - // if it exists, to be promoted to the primary cache. The helper may be - // saved and used later when the object is evicted. Therefore, it must - // outlive the cache. + // If a non-nullptr helper argument is provided with a non-nullptr + // create_cb, and a secondary cache is configured, then the secondary + // cache is also queried if lookup in the primary cache fails. If found + // in secondary cache, the provided create_db and create_context are + // used to promote the entry to an object in the primary cache. + // In that case, the helper may be saved and used later when the object + // is evicted, so as usual, the pointed-to helper must outlive the cache. // // ======================== Async Lookup (wait=false) ====================== // When wait=false, the handle returned might be in any of three states: @@ -576,8 +583,8 @@ class Cache { // * Pending, not ready (IsReady() == false) - secondary cache is still // working to retrieve the value. Might become ready any time. // * Pending, ready (IsReady() == true) - secondary cache has the value - // but it has not been loaded into primary cache. Call to Wait()/WaitAll() - // will not block. + // but it has not been loaded as an object into primary cache. Call to + // Wait()/WaitAll() will not block. // // IMPORTANT: Pending handles are not thread-safe, and only these functions // are allowed on them: Value(), IsReady(), Wait(), WaitAll(). Even Release() @@ -594,11 +601,15 @@ class Cache { // Pending+ready state from the Failed state is to Wait() on it. A cache // entry not compatible with secondary cache can also have Value()==nullptr // like the Failed state, but this is not generally a concern. - virtual Handle* Lookup(const Slice& key, const CacheItemHelper* /*helper_cb*/, - const CreateCallback& /*create_cb*/, - Priority /*priority*/, bool /*wait*/, - Statistics* stats = nullptr) { - return Lookup(key, stats); + virtual Handle* Lookup(const Slice& key, + const CacheItemHelper* helper = nullptr, + CreateContext* create_context = nullptr, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr) = 0; + + // Convenience wrapper when secondary cache not supported + inline Handle* BasicLookup(const Slice& key, Statistics* stats) { + return Lookup(key, nullptr, nullptr, Priority::LOW, true, stats); } // Increments the reference count for the handle if it refers to an entry in @@ -620,11 +631,12 @@ class Cache { // REQUIRES: handle must have been returned by a method on *this. virtual bool Release(Handle* handle, bool erase_if_last_ref = false) = 0; - // Return the value encapsulated in a handle returned by a - // successful Lookup(). + // Return the object assiciated with a handle returned by a successful + // Lookup(). For historical reasons, this is also known at the "value" + // associated with the key. // REQUIRES: handle must not have been released yet. // REQUIRES: handle must have been returned by a method on *this. - virtual void* Value(Handle* handle) = 0; + virtual ObjectPtr Value(Handle* handle) = 0; // If the cache contains the entry for the key, erase it. Note that the // underlying entry will be kept around until all existing handles @@ -675,11 +687,8 @@ class Cache { // Returns the charge for the specific entry in the cache. virtual size_t GetCharge(Handle* handle) const = 0; - // Returns the deleter for the specified entry. This might seem useless - // as the Cache itself is responsible for calling the deleter, but - // the deleter can essentially verify that a cache entry is of an - // expected type from an expected code source. - virtual DeleterFn GetDeleter(Handle* handle) const = 0; + // Returns the helper for the specified entry. + virtual const CacheItemHelper* GetCacheItemHelper(Handle* handle) const = 0; // Call this on shutdown if you want to speed it up. Cache will disown // any underlying data and will not free it on delete. This call will leak @@ -705,19 +714,10 @@ class Cache { // entries is iterated over if other threads are operating on the Cache // also. virtual void ApplyToAllEntries( - const std::function& callback, + const std::function& callback, const ApplyToAllEntriesOptions& opts) = 0; - // DEPRECATED version of above. (Default implementation uses above.) - virtual void ApplyToAllCacheEntries(void (*callback)(void* value, - size_t charge), - bool /*thread_safe*/) { - ApplyToAllEntries([callback](const Slice&, void* value, size_t charge, - DeleterFn) { callback(value, charge); }, - {}); - } - // Remove all entries. // Prerequisite: no entry is referenced. virtual void EraseUnRefEntries() = 0; @@ -734,6 +734,8 @@ class Cache { MemoryAllocator* memory_allocator() const { return memory_allocator_.get(); } // EXPERIMENTAL + // The following APIs are experimental and might change in the future. + // Release a mapping returned by a previous Lookup(). The "useful" // parameter specifies whether the data was actually used or not, // which may be used by the cache implementation to decide whether @@ -744,24 +746,21 @@ class Cache { return Release(handle, erase_if_last_ref); } - // EXPERIMENTAL // Determines if the handle returned by Lookup() can give a value without // blocking, though Wait()/WaitAll() might be required to publish it to // Value(). See secondary cache compatible Lookup() above for details. // This call is not thread safe on "pending" handles. virtual bool IsReady(Handle* /*handle*/) { return true; } - // EXPERIMENTAL // Convert a "pending" handle into a full thread-shareable handle by // * If necessary, wait until secondary cache finishes loading the value. - // * Construct the value for primary cache and set it in the handle. + // * Construct the object for primary cache and set it in the handle. // Even after Wait() on a pending handle, the caller must check for // Value() == nullptr in case of failure. This call is not thread-safe // on pending handles. This call has no effect on non-pending handles. // See secondary cache compatible Lookup() above for details. virtual void Wait(Handle* /*handle*/) {} - // EXPERIMENTAL // Wait for a vector of handles to become ready. As with Wait(), the user // should check the Value() of each handle for nullptr. This call is not // thread-safe on pending handles. @@ -771,5 +770,8 @@ class Cache { std::shared_ptr memory_allocator_; }; +// Useful for cache entries requiring no clean-up, such as for cache +// reservations +inline constexpr Cache::CacheItemHelper kNoopCacheItemHelper{}; } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/secondary_cache.h b/include/rocksdb/secondary_cache.h index a6a8c8b1d5..cb6f74450a 100644 --- a/include/rocksdb/secondary_cache.h +++ b/include/rocksdb/secondary_cache.h @@ -20,7 +20,7 @@ namespace ROCKSDB_NAMESPACE { // A handle for lookup result. The handle may not be immediately ready or // have a valid value. The caller must call isReady() to determine if its // ready, and call Wait() in order to block until it becomes ready. -// The caller must call value() after it becomes ready to determine if the +// The caller must call Value() after it becomes ready to determine if the // handle successfullly read the item. class SecondaryCacheResultHandle { public: @@ -32,8 +32,9 @@ class SecondaryCacheResultHandle { // Block until handle becomes ready virtual void Wait() = 0; - // Return the value. If nullptr, it means the lookup was unsuccessful - virtual void* Value() = 0; + // Return the cache entry object (also known as value). If nullptr, it means + // the lookup was unsuccessful. + virtual Cache::ObjectPtr Value() = 0; // Return the size of value virtual size_t Size() = 0; @@ -74,7 +75,7 @@ class SecondaryCache : public Customizable { // Lookup() might return the same parsed value back. But more typically, if // the implementation only uses `value` for getting persistable data during // the call, then the default implementation of `InsertSaved()` suffices. - virtual Status Insert(const Slice& key, void* value, + virtual Status Insert(const Slice& key, Cache::ObjectPtr obj, const Cache::CacheItemHelper* helper) = 0; // Insert a value from its saved/persistable data (typically uncompressed @@ -101,8 +102,9 @@ class SecondaryCache : public Customizable { // is_in_sec_cache is to indicate whether the handle is possibly erased // from the secondary cache after the Lookup. virtual std::unique_ptr Lookup( - const Slice& key, const Cache::CreateCallback& create_cb, bool wait, - bool advise_erase, bool& is_in_sec_cache) = 0; + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, + bool& is_in_sec_cache) = 0; // Indicate whether a handle can be erased in this secondary cache. [[nodiscard]] virtual bool SupportForceErase() const = 0; diff --git a/memory/memory_allocator.h b/memory/memory_allocator.h index f1a548659b..68aa35beb8 100644 --- a/memory/memory_allocator.h +++ b/memory/memory_allocator.h @@ -6,6 +6,8 @@ #pragma once +#include + #include "rocksdb/memory_allocator.h" namespace ROCKSDB_NAMESPACE { @@ -35,4 +37,11 @@ inline CacheAllocationPtr AllocateBlock(size_t size, return CacheAllocationPtr(new char[size]); } +inline CacheAllocationPtr AllocateAndCopyBlock(const Slice& data, + MemoryAllocator* allocator) { + CacheAllocationPtr cap = AllocateBlock(data.size(), allocator); + std::copy_n(data.data(), data.size(), cap.get()); + return cap; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 9d3c86c620..2ed4eeb9e7 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -1324,13 +1324,14 @@ class TestSecondaryCache : public SecondaryCache { public: static const char* kClassName() { return "Test"; } const char* Name() const override { return kClassName(); } - Status Insert(const Slice& /*key*/, void* /*value*/, + Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, const Cache::CacheItemHelper* /*helper*/) override { return Status::NotSupported(); } std::unique_ptr Lookup( - const Slice& /*key*/, const Cache::CreateCallback& /*create_cb*/, - bool /*wait*/, bool /*advise_erase*/, bool& is_in_sec_cache) override { + const Slice& /*key*/, const Cache::CacheItemHelper* /*helper*/, + Cache::CreateContext* /*create_context*/, bool /*wait*/, + bool /*advise_erase*/, bool& is_in_sec_cache) override { is_in_sec_cache = true; return nullptr; } diff --git a/src.mk b/src.mk index ebc95a6599..79f57c738c 100644 --- a/src.mk +++ b/src.mk @@ -3,6 +3,7 @@ LIB_SOURCES = \ cache/cache.cc \ cache/cache_entry_roles.cc \ cache/cache_key.cc \ + cache/cache_helpers.cc \ cache/cache_reservation_manager.cc \ cache/charged_cache.cc \ cache/clock_cache.cc \ @@ -171,6 +172,7 @@ LIB_SOURCES = \ table/block_based/block_based_table_iterator.cc \ table/block_based/block_based_table_reader.cc \ table/block_based/block_builder.cc \ + table/block_based/block_cache.cc \ table/block_based/block_prefetcher.cc \ table/block_based/block_prefix_index.cc \ table/block_based/data_block_hash_index.cc \ diff --git a/table/block_based/block.h b/table/block_based/block.h index 5d73f72f6a..90f9aa397b 100644 --- a/table/block_based/block.h +++ b/table/block_based/block.h @@ -236,6 +236,9 @@ class Block { // Report an approximation of how much memory has been used. size_t ApproximateMemoryUsage() const; + // For TypedCacheInterface + const Slice& ContentSlice() const { return contents_.data; } + private: BlockContents contents_; const char* data_; // contents_.data.data() diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index fed69af072..81113c9c7a 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -21,6 +21,7 @@ #include #include +#include "block_cache.h" #include "cache/cache_entry_roles.h" #include "cache/cache_helpers.h" #include "cache/cache_key.h" @@ -41,7 +42,6 @@ #include "table/block_based/block_based_table_factory.h" #include "table/block_based/block_based_table_reader.h" #include "table/block_based/block_builder.h" -#include "table/block_based/block_like_traits.h" #include "table/block_based/filter_block.h" #include "table/block_based/filter_policy_internal.h" #include "table/block_based/full_filter_block.h" @@ -335,6 +335,7 @@ struct BlockBasedTableBuilder::Rep { std::vector> table_properties_collectors; std::unique_ptr pc_rep; + BlockCreateContext create_context; uint64_t get_offset() { return offset.load(std::memory_order_relaxed); } void set_offset(uint64_t o) { offset.store(o, std::memory_order_relaxed); } @@ -443,6 +444,9 @@ struct BlockBasedTableBuilder::Rep { flush_block_policy( table_options.flush_block_policy_factory->NewFlushBlockPolicy( table_options, data_block)), + create_context(&table_options, ioptions.stats, + compression_type == kZSTD || + compression_type == kZSTDNotFinalCompression), status_ok(true), io_status_ok(true) { if (tbo.target_file_size == 0) { @@ -1240,6 +1244,10 @@ void BlockBasedTableBuilder::WriteMaybeCompressedBlock( handle->set_size(block_contents.size()); assert(status().ok()); assert(io_status().ok()); + if (uncompressed_block_data == nullptr) { + uncompressed_block_data = &block_contents; + assert(type == kNoCompression); + } { IOStatus io_s = r->file->Append(block_contents); @@ -1291,12 +1299,8 @@ void BlockBasedTableBuilder::WriteMaybeCompressedBlock( warm_cache = false; } if (warm_cache) { - if (type == kNoCompression) { - s = InsertBlockInCacheHelper(block_contents, handle, block_type); - } else if (uncompressed_block_data != nullptr) { - s = InsertBlockInCacheHelper(*uncompressed_block_data, handle, - block_type); - } + s = InsertBlockInCacheHelper(*uncompressed_block_data, handle, + block_type); if (!s.ok()) { r->SetStatus(s); return; @@ -1425,13 +1429,14 @@ Status BlockBasedTableBuilder::InsertBlockInCompressedCache( const Slice& block_contents, const CompressionType type, const BlockHandle* handle) { Rep* r = rep_; - Cache* block_cache_compressed = r->table_options.block_cache_compressed.get(); + CompressedBlockCacheInterface block_cache_compressed{ + r->table_options.block_cache_compressed.get()}; Status s; - if (type != kNoCompression && block_cache_compressed != nullptr) { + if (type != kNoCompression && block_cache_compressed) { size_t size = block_contents.size(); - auto ubuf = - AllocateBlock(size + 1, block_cache_compressed->memory_allocator()); + auto ubuf = AllocateBlock(size + 1, + block_cache_compressed.get()->memory_allocator()); memcpy(ubuf.get(), block_contents.data(), size); ubuf[size] = type; @@ -1443,10 +1448,9 @@ Status BlockBasedTableBuilder::InsertBlockInCompressedCache( CacheKey key = BlockBasedTable::GetCacheKey(rep_->base_cache_key, *handle); - s = block_cache_compressed->Insert( + s = block_cache_compressed.Insert( key.AsSlice(), block_contents_to_cache, - block_contents_to_cache->ApproximateMemoryUsage(), - &DeleteCacheEntry); + block_contents_to_cache->ApproximateMemoryUsage()); if (s.ok()) { RecordTick(rep_->ioptions.stats, BLOCK_CACHE_COMPRESSED_ADD); } else { @@ -1462,65 +1466,19 @@ Status BlockBasedTableBuilder::InsertBlockInCompressedCache( Status BlockBasedTableBuilder::InsertBlockInCacheHelper( const Slice& block_contents, const BlockHandle* handle, BlockType block_type) { - Status s; - switch (block_type) { - case BlockType::kData: - case BlockType::kIndex: - case BlockType::kFilterPartitionIndex: - s = InsertBlockInCache(block_contents, handle, block_type); - break; - case BlockType::kFilter: - s = InsertBlockInCache(block_contents, handle, - block_type); - break; - case BlockType::kCompressionDictionary: - s = InsertBlockInCache(block_contents, handle, - block_type); - break; - default: - // no-op / not cached - break; - } - return s; -} -template -Status BlockBasedTableBuilder::InsertBlockInCache(const Slice& block_contents, - const BlockHandle* handle, - BlockType block_type) { - // Uncompressed regular block cache Cache* block_cache = rep_->table_options.block_cache.get(); Status s; - if (block_cache != nullptr) { - size_t size = block_contents.size(); - auto buf = AllocateBlock(size, block_cache->memory_allocator()); - memcpy(buf.get(), block_contents.data(), size); - BlockContents results(std::move(buf), size); - + auto helper = + GetCacheItemHelper(block_type, rep_->ioptions.lowest_used_cache_tier); + if (block_cache && helper && helper->create_cb) { CacheKey key = BlockBasedTable::GetCacheKey(rep_->base_cache_key, *handle); - - const size_t read_amp_bytes_per_bit = - rep_->table_options.read_amp_bytes_per_bit; - - // TODO akanksha:: Dedup below code by calling - // BlockBasedTable::PutDataBlockToCache. - std::unique_ptr block_holder( - BlocklikeTraits::Create( - std::move(results), read_amp_bytes_per_bit, - rep_->ioptions.statistics.get(), - false /*rep_->blocks_definitely_zstd_compressed*/, - rep_->table_options.filter_policy.get())); - - assert(block_holder->own_bytes()); - size_t charge = block_holder->ApproximateMemoryUsage(); - s = block_cache->Insert( - key.AsSlice(), block_holder.get(), - BlocklikeTraits::GetCacheItemHelper(block_type), charge, - nullptr, Cache::Priority::LOW); + size_t charge; + s = WarmInCache(block_cache, key.AsSlice(), block_contents, + &rep_->create_context, helper, Cache::Priority::LOW, + &charge); if (s.ok()) { - // Release ownership of block_holder. - block_holder.release(); BlockBasedTable::UpdateCacheInsertionMetrics( block_type, nullptr /*get_context*/, charge, s.IsOkOverwritten(), rep_->ioptions.stats); diff --git a/table/block_based/block_based_table_builder.h b/table/block_based/block_based_table_builder.h index ecc13d0f7d..7cf33953a1 100644 --- a/table/block_based/block_based_table_builder.h +++ b/table/block_based/block_based_table_builder.h @@ -122,9 +122,9 @@ class BlockBasedTableBuilder : public TableBuilder { void WriteBlock(const Slice& block_contents, BlockHandle* handle, BlockType block_type); // Directly write data to the file. - void WriteMaybeCompressedBlock(const Slice& data, CompressionType, - BlockHandle* handle, BlockType block_type, - const Slice* raw_data = nullptr); + void WriteMaybeCompressedBlock( + const Slice& block_contents, CompressionType, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data = nullptr); void SetupCacheKeyPrefix(const TableBuilderOptions& tbo); diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 09c1d2f623..b3f76a731a 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -525,20 +525,24 @@ Status CheckCacheOptionCompatibility(const BlockBasedTableOptions& bbto) { // More complex test of shared key space, in case the instances are wrappers // for some shared underlying cache. + static Cache::CacheItemHelper kHelper{CacheEntryRole::kMisc}; CacheKey sentinel_key = CacheKey::CreateUniqueForProcessLifetime(); - static char kRegularBlockCacheMarker = 'b'; - static char kCompressedBlockCacheMarker = 'c'; - static char kPersistentCacheMarker = 'p'; + struct SentinelValue { + explicit SentinelValue(char _c) : c(_c) {} + char c; + }; + static SentinelValue kRegularBlockCacheMarker{'b'}; + static SentinelValue kCompressedBlockCacheMarker{'c'}; + static char kPersistentCacheMarker{'p'}; if (bbto.block_cache) { bbto.block_cache - ->Insert(sentinel_key.AsSlice(), &kRegularBlockCacheMarker, 1, - GetNoopDeleterForRole()) + ->Insert(sentinel_key.AsSlice(), &kRegularBlockCacheMarker, &kHelper, 1) .PermitUncheckedError(); } if (bbto.block_cache_compressed) { bbto.block_cache_compressed - ->Insert(sentinel_key.AsSlice(), &kCompressedBlockCacheMarker, 1, - GetNoopDeleterForRole()) + ->Insert(sentinel_key.AsSlice(), &kCompressedBlockCacheMarker, &kHelper, + 1) .PermitUncheckedError(); } if (bbto.persistent_cache) { @@ -552,8 +556,8 @@ Status CheckCacheOptionCompatibility(const BlockBasedTableOptions& bbto) { if (bbto.block_cache) { auto handle = bbto.block_cache->Lookup(sentinel_key.AsSlice()); if (handle) { - auto v = static_cast(bbto.block_cache->Value(handle)); - char c = *v; + auto v = static_cast(bbto.block_cache->Value(handle)); + char c = v->c; bbto.block_cache->Release(handle); if (v == &kCompressedBlockCacheMarker) { return Status::InvalidArgument( @@ -571,8 +575,9 @@ Status CheckCacheOptionCompatibility(const BlockBasedTableOptions& bbto) { if (bbto.block_cache_compressed) { auto handle = bbto.block_cache_compressed->Lookup(sentinel_key.AsSlice()); if (handle) { - auto v = static_cast(bbto.block_cache_compressed->Value(handle)); - char c = *v; + auto v = static_cast( + bbto.block_cache_compressed->Value(handle)); + char c = v->c; bbto.block_cache_compressed->Release(handle); if (v == &kRegularBlockCacheMarker) { return Status::InvalidArgument( @@ -595,11 +600,11 @@ Status CheckCacheOptionCompatibility(const BlockBasedTableOptions& bbto) { bbto.persistent_cache->Lookup(sentinel_key.AsSlice(), &data, &size) .PermitUncheckedError(); if (data && size > 0) { - if (data[0] == kRegularBlockCacheMarker) { + if (data[0] == kRegularBlockCacheMarker.c) { return Status::InvalidArgument( "persistent_cache and block_cache share the same key space, " "which is not supported"); - } else if (data[0] == kCompressedBlockCacheMarker) { + } else if (data[0] == kCompressedBlockCacheMarker.c) { return Status::InvalidArgument( "persistent_cache and block_cache_compressed share the same key " "space, " diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index a644129905..e9005cbac0 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -19,6 +19,7 @@ #include #include +#include "block_cache.h" #include "cache/cache_entry_roles.h" #include "cache/cache_key.h" #include "db/compaction/compaction_picker.h" @@ -29,6 +30,7 @@ #include "file/random_access_file_reader.h" #include "logging/logging.h" #include "monitoring/perf_context_imp.h" +#include "parsed_full_filter_block.h" #include "port/lang.h" #include "rocksdb/cache.h" #include "rocksdb/comparator.h" @@ -48,7 +50,6 @@ #include "table/block_based/block.h" #include "table/block_based/block_based_table_factory.h" #include "table/block_based/block_based_table_iterator.h" -#include "table/block_based/block_like_traits.h" #include "table/block_based/block_prefix_index.h" #include "table/block_based/block_type.h" #include "table/block_based/filter_block.h" @@ -83,6 +84,26 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) { return heap_buf; } } // namespace + +// Explicitly instantiate templates for each "blocklike" type we use (and +// before implicit specialization). +// This makes it possible to keep the template definitions in the .cc file. +#define INSTANTIATE_RETRIEVE_BLOCK(T) \ + template Status BlockBasedTable::RetrieveBlock( \ + FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \ + const BlockHandle& handle, const UncompressionDict& uncompression_dict, \ + CachableEntry* out_parsed_block, GetContext* get_context, \ + BlockCacheLookupContext* lookup_context, bool for_compaction, \ + bool use_cache, bool wait_for_cache, bool async_read) const; + +INSTANTIATE_RETRIEVE_BLOCK(ParsedFullFilterBlock); +INSTANTIATE_RETRIEVE_BLOCK(UncompressionDict); +INSTANTIATE_RETRIEVE_BLOCK(Block_kData); +INSTANTIATE_RETRIEVE_BLOCK(Block_kIndex); +INSTANTIATE_RETRIEVE_BLOCK(Block_kFilterPartitionIndex); +INSTANTIATE_RETRIEVE_BLOCK(Block_kRangeDeletion); +INSTANTIATE_RETRIEVE_BLOCK(Block_kMetaIndex); + } // namespace ROCKSDB_NAMESPACE // Generate the regular and coroutine versions of some methods by @@ -114,22 +135,22 @@ namespace { // @param uncompression_dict Data for presetting the compression library's // dictionary. template -Status ReadBlockFromFile( +Status ReadAndParseBlockFromFile( RandomAccessFileReader* file, FilePrefetchBuffer* prefetch_buffer, const Footer& footer, const ReadOptions& options, const BlockHandle& handle, std::unique_ptr* result, const ImmutableOptions& ioptions, - bool do_uncompress, bool maybe_compressed, BlockType block_type, + BlockCreateContext& create_context, bool maybe_compressed, const UncompressionDict& uncompression_dict, - const PersistentCacheOptions& cache_options, size_t read_amp_bytes_per_bit, - MemoryAllocator* memory_allocator, bool for_compaction, bool using_zstd, - const FilterPolicy* filter_policy, bool async_read) { + const PersistentCacheOptions& cache_options, + MemoryAllocator* memory_allocator, bool for_compaction, bool async_read) { assert(result); BlockContents contents; BlockFetcher block_fetcher( file, prefetch_buffer, footer, options, handle, &contents, ioptions, - do_uncompress, maybe_compressed, block_type, uncompression_dict, - cache_options, memory_allocator, nullptr, for_compaction); + /*do_uncompress*/ maybe_compressed, maybe_compressed, + TBlocklike::kBlockType, uncompression_dict, cache_options, + memory_allocator, nullptr, for_compaction); Status s; // If prefetch_buffer is not allocated, it will fallback to synchronous // reading of block contents. @@ -142,11 +163,8 @@ Status ReadBlockFromFile( s = block_fetcher.ReadBlockContents(); } if (s.ok()) { - result->reset(BlocklikeTraits::Create( - std::move(contents), read_amp_bytes_per_bit, ioptions.stats, using_zstd, - filter_policy)); + create_context.Create(result, std::move(contents)); } - return s; } @@ -171,6 +189,16 @@ inline bool PrefixExtractorChangedHelper( } } +template +uint32_t GetBlockNumRestarts(const TBlocklike& block) { + if constexpr (std::is_convertible_v) { + const Block& b = block; + return b.NumRestarts(); + } else { + return 0; + } +} + } // namespace void BlockBasedTable::UpdateCacheHitMetrics(BlockType block_type, @@ -377,56 +405,6 @@ void BlockBasedTable::UpdateCacheInsertionMetrics( } } -Cache::Handle* BlockBasedTable::GetEntryFromCache( - const CacheTier& cache_tier, Cache* block_cache, const Slice& key, - BlockType block_type, const bool wait, GetContext* get_context, - const Cache::CacheItemHelper* cache_helper, - const Cache::CreateCallback& create_cb, Cache::Priority priority) const { - Cache::Handle* cache_handle = nullptr; - if (cache_tier == CacheTier::kNonVolatileBlockTier) { - cache_handle = block_cache->Lookup(key, cache_helper, create_cb, priority, - wait, rep_->ioptions.statistics.get()); - } else { - cache_handle = block_cache->Lookup(key, rep_->ioptions.statistics.get()); - } - - // Avoid updating metrics here if the handle is not complete yet. This - // happens with MultiGet and secondary cache. So update the metrics only - // if its a miss, or a hit and value is ready - if (!cache_handle || block_cache->Value(cache_handle)) { - if (cache_handle != nullptr) { - UpdateCacheHitMetrics(block_type, get_context, - block_cache->GetUsage(cache_handle)); - } else { - UpdateCacheMissMetrics(block_type, get_context); - } - } - - return cache_handle; -} - -template -Status BlockBasedTable::InsertEntryToCache( - const CacheTier& cache_tier, Cache* block_cache, const Slice& key, - const Cache::CacheItemHelper* cache_helper, - std::unique_ptr&& block_holder, size_t charge, - Cache::Handle** cache_handle, Cache::Priority priority) const { - Status s = Status::OK(); - if (cache_tier == CacheTier::kNonVolatileBlockTier) { - s = block_cache->Insert(key, block_holder.get(), cache_helper, charge, - cache_handle, priority); - } else { - s = block_cache->Insert(key, block_holder.get(), charge, - cache_helper->del_cb, cache_handle, priority); - } - if (s.ok()) { - // Cache took ownership - block_holder.release(); - } - s.MustCheck(); - return s; -} - namespace { // Return True if table_properties has `user_prop_name` has a `true` value // or it doesn't contain this property (for backward compatible). @@ -687,6 +665,17 @@ Status BlockBasedTable::Open( return s; } + // Populate BlockCreateContext + bool blocks_definitely_zstd_compressed = + rep->table_properties && + (rep->table_properties->compression_name == + CompressionTypeToString(kZSTD) || + rep->table_properties->compression_name == + CompressionTypeToString(kZSTDNotFinalCompression)); + rep->create_context = + BlockCreateContext(&rep->table_options, rep->ioptions.stats, + blocks_definitely_zstd_compressed); + // Check expected unique id if provided if (expected_unique_id != kNullUniqueId64x2) { auto props = rep->table_properties; @@ -903,11 +892,6 @@ Status BlockBasedTable::ReadPropertiesBlock( rep_->blocks_maybe_compressed = rep_->table_properties->compression_name != CompressionTypeToString(kNoCompression); - rep_->blocks_definitely_zstd_compressed = - (rep_->table_properties->compression_name == - CompressionTypeToString(kZSTD) || - rep_->table_properties->compression_name == - CompressionTypeToString(kZSTDNotFinalCompression)); } } else { ROCKS_LOG_ERROR(rep_->ioptions.logger, @@ -1247,15 +1231,14 @@ Status BlockBasedTable::ReadMetaIndexBlock( std::unique_ptr* iter) { // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates // it is an empty block. - std::unique_ptr metaindex; - Status s = ReadBlockFromFile( + std::unique_ptr metaindex; + Status s = ReadAndParseBlockFromFile( rep_->file.get(), prefetch_buffer, rep_->footer, ro, rep_->footer.metaindex_handle(), &metaindex, rep_->ioptions, - true /* decompress */, true /*maybe_compressed*/, BlockType::kMetaIndex, + rep_->create_context, true /*maybe_compressed*/, UncompressionDict::GetEmptyDict(), rep_->persistent_cache_options, - 0 /* read_amp_bytes_per_bit */, GetMemoryAllocator(rep_->table_options), - false /* for_compaction */, rep_->blocks_definitely_zstd_compressed, - nullptr /* filter_policy */, false /* async_read */); + GetMemoryAllocator(rep_->table_options), false /* for_compaction */, + false /* async_read */); if (!s.ok()) { ROCKS_LOG_ERROR(rep_->ioptions.logger, @@ -1272,16 +1255,13 @@ Status BlockBasedTable::ReadMetaIndexBlock( } template -Status BlockBasedTable::GetDataBlockFromCache( - const Slice& cache_key, Cache* block_cache, Cache* block_cache_compressed, +WithBlocklikeCheck BlockBasedTable::GetDataBlockFromCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CompressedBlockCacheInterface block_cache_compressed, const ReadOptions& read_options, CachableEntry* out_parsed_block, - const UncompressionDict& uncompression_dict, BlockType block_type, - const bool wait, GetContext* get_context) const { - const size_t read_amp_bytes_per_bit = - block_type == BlockType::kData - ? rep_->table_options.read_amp_bytes_per_bit - : 0; + const UncompressionDict& uncompression_dict, const bool wait, + GetContext* get_context) const { assert(out_parsed_block); assert(out_parsed_block->IsEmpty()); // Here we treat the legacy name "...index_and_filter_blocks..." to mean all @@ -1292,33 +1272,33 @@ Status BlockBasedTable::GetDataBlockFromCache( // high-priority treatment if it should go into BlockCache. const Cache::Priority priority = rep_->table_options.cache_index_and_filter_blocks_with_high_priority && - block_type != BlockType::kData && - block_type != BlockType::kProperties + TBlocklike::kBlockType != BlockType::kData && + TBlocklike::kBlockType != BlockType::kProperties ? Cache::Priority::HIGH : Cache::Priority::LOW; Status s; - BlockContents* compressed_block = nullptr; - Cache::Handle* block_cache_compressed_handle = nullptr; Statistics* statistics = rep_->ioptions.statistics.get(); - bool using_zstd = rep_->blocks_definitely_zstd_compressed; - const FilterPolicy* filter_policy = rep_->filter_policy; - Cache::CreateCallback create_cb = GetCreateCallback( - read_amp_bytes_per_bit, statistics, using_zstd, filter_policy); // Lookup uncompressed cache first - if (block_cache != nullptr) { + if (block_cache) { assert(!cache_key.empty()); - Cache::Handle* cache_handle = nullptr; - cache_handle = GetEntryFromCache( - rep_->ioptions.lowest_used_cache_tier, block_cache, cache_key, - block_type, wait, get_context, - BlocklikeTraits::GetCacheItemHelper(block_type), create_cb, - priority); - if (cache_handle != nullptr) { - out_parsed_block->SetCachedValue( - reinterpret_cast(block_cache->Value(cache_handle)), - block_cache, cache_handle); + auto cache_handle = block_cache.LookupFull( + cache_key, &rep_->create_context, priority, wait, statistics, + rep_->ioptions.lowest_used_cache_tier); + + // Avoid updating metrics here if the handle is not complete yet. This + // happens with MultiGet and secondary cache. So update the metrics only + // if its a miss, or a hit and value is ready + if (!cache_handle) { + UpdateCacheMissMetrics(TBlocklike::kBlockType, get_context); + } else { + TBlocklike* value = block_cache.Value(cache_handle); + if (value) { + UpdateCacheHitMetrics(TBlocklike::kBlockType, get_context, + block_cache.get()->GetUsage(cache_handle)); + } + out_parsed_block->SetCachedValue(value, block_cache.get(), cache_handle); return s; } } @@ -1326,14 +1306,14 @@ Status BlockBasedTable::GetDataBlockFromCache( // If not found, search from the compressed block cache. assert(out_parsed_block->IsEmpty()); - if (block_cache_compressed == nullptr) { + if (!block_cache_compressed) { return s; } assert(!cache_key.empty()); BlockContents contents; - block_cache_compressed_handle = - block_cache_compressed->Lookup(cache_key, statistics); + auto block_cache_compressed_handle = + block_cache_compressed.Lookup(cache_key, statistics); // if we found in the compressed cache, then uncompress and insert into // uncompressed cache @@ -1344,8 +1324,8 @@ Status BlockBasedTable::GetDataBlockFromCache( // found compressed block RecordTick(statistics, BLOCK_CACHE_COMPRESSED_HIT); - compressed_block = reinterpret_cast( - block_cache_compressed->Value(block_cache_compressed_handle)); + BlockContents* compressed_block = + block_cache_compressed.Value(block_cache_compressed_handle); CompressionType compression_type = GetBlockCompressionType(*compressed_block); assert(compression_type != kNoCompression); @@ -1360,27 +1340,21 @@ Status BlockBasedTable::GetDataBlockFromCache( // Insert parsed block into block cache, the priority is based on the // data block type. if (s.ok()) { - std::unique_ptr block_holder( - BlocklikeTraits::Create( - std::move(contents), read_amp_bytes_per_bit, statistics, - rep_->blocks_definitely_zstd_compressed, - rep_->table_options.filter_policy.get())); + std::unique_ptr block_holder; + rep_->create_context.Create(&block_holder, std::move(contents)); - if (block_cache != nullptr && block_holder->own_bytes() && - read_options.fill_cache) { + if (block_cache && block_holder->own_bytes() && read_options.fill_cache) { size_t charge = block_holder->ApproximateMemoryUsage(); - Cache::Handle* cache_handle = nullptr; - auto block_holder_raw_ptr = block_holder.get(); - s = InsertEntryToCache( - rep_->ioptions.lowest_used_cache_tier, block_cache, cache_key, - BlocklikeTraits::GetCacheItemHelper(block_type), - std::move(block_holder), charge, &cache_handle, priority); + BlockCacheTypedHandle* cache_handle = nullptr; + s = block_cache.InsertFull(cache_key, block_holder.get(), charge, + &cache_handle, priority, + rep_->ioptions.lowest_used_cache_tier); if (s.ok()) { assert(cache_handle != nullptr); - out_parsed_block->SetCachedValue(block_holder_raw_ptr, block_cache, - cache_handle); + out_parsed_block->SetCachedValue(block_holder.release(), + block_cache.get(), cache_handle); - UpdateCacheInsertionMetrics(block_type, get_context, charge, + UpdateCacheInsertionMetrics(TBlocklike::kBlockType, get_context, charge, s.IsOkOverwritten(), rep_->ioptions.stats); } else { RecordTick(statistics, BLOCK_CACHE_ADD_FAILURES); @@ -1391,27 +1365,23 @@ Status BlockBasedTable::GetDataBlockFromCache( } // Release hold on compressed cache entry - block_cache_compressed->Release(block_cache_compressed_handle); + block_cache_compressed.Release(block_cache_compressed_handle); return s; } template -Status BlockBasedTable::PutDataBlockToCache( - const Slice& cache_key, Cache* block_cache, Cache* block_cache_compressed, +WithBlocklikeCheck BlockBasedTable::PutDataBlockToCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CompressedBlockCacheInterface block_cache_compressed, CachableEntry* out_parsed_block, BlockContents&& block_contents, CompressionType block_comp_type, const UncompressionDict& uncompression_dict, - MemoryAllocator* memory_allocator, BlockType block_type, - GetContext* get_context) const { + MemoryAllocator* memory_allocator, GetContext* get_context) const { const ImmutableOptions& ioptions = rep_->ioptions; const uint32_t format_version = rep_->table_options.format_version; - const size_t read_amp_bytes_per_bit = - block_type == BlockType::kData - ? rep_->table_options.read_amp_bytes_per_bit - : 0; const Cache::Priority priority = rep_->table_options.cache_index_and_filter_blocks_with_high_priority && - block_type != BlockType::kData + TBlocklike::kBlockType != BlockType::kData ? Cache::Priority::HIGH : Cache::Priority::LOW; assert(out_parsed_block); @@ -1433,21 +1403,15 @@ Status BlockBasedTable::PutDataBlockToCache( if (!s.ok()) { return s; } - - block_holder.reset(BlocklikeTraits::Create( - std::move(uncompressed_block_contents), read_amp_bytes_per_bit, - statistics, rep_->blocks_definitely_zstd_compressed, - rep_->table_options.filter_policy.get())); + rep_->create_context.Create(&block_holder, + std::move(uncompressed_block_contents)); } else { - block_holder.reset(BlocklikeTraits::Create( - std::move(block_contents), read_amp_bytes_per_bit, statistics, - rep_->blocks_definitely_zstd_compressed, - rep_->table_options.filter_policy.get())); + rep_->create_context.Create(&block_holder, std::move(block_contents)); } // Insert compressed block into compressed block cache. // Release the hold on the compressed cache entry immediately. - if (block_cache_compressed != nullptr && block_comp_type != kNoCompression && + if (block_cache_compressed && block_comp_type != kNoCompression && block_contents.own_bytes()) { assert(block_contents.has_trailer); assert(!cache_key.empty()); @@ -1458,10 +1422,9 @@ Status BlockBasedTable::PutDataBlockToCache( std::make_unique(std::move(block_contents)); size_t charge = block_cont_for_comp_cache->ApproximateMemoryUsage(); - s = block_cache_compressed->Insert( - cache_key, block_cont_for_comp_cache.get(), charge, - &DeleteCacheEntry, nullptr /*handle*/, - Cache::Priority::LOW); + s = block_cache_compressed.Insert(cache_key, + block_cont_for_comp_cache.get(), charge, + nullptr /*handle*/, Cache::Priority::LOW); if (s.ok()) { // Cache took ownership @@ -1473,20 +1436,19 @@ Status BlockBasedTable::PutDataBlockToCache( } // insert into uncompressed block cache - if (block_cache != nullptr && block_holder->own_bytes()) { + if (block_cache && block_holder->own_bytes()) { size_t charge = block_holder->ApproximateMemoryUsage(); - auto block_holder_raw_ptr = block_holder.get(); - Cache::Handle* cache_handle = nullptr; - s = InsertEntryToCache( - rep_->ioptions.lowest_used_cache_tier, block_cache, cache_key, - BlocklikeTraits::GetCacheItemHelper(block_type), - std::move(block_holder), charge, &cache_handle, priority); + BlockCacheTypedHandle* cache_handle = nullptr; + s = block_cache.InsertFull(cache_key, block_holder.get(), charge, + &cache_handle, priority, + rep_->ioptions.lowest_used_cache_tier); + if (s.ok()) { assert(cache_handle != nullptr); - out_parsed_block->SetCachedValue(block_holder_raw_ptr, block_cache, - cache_handle); + out_parsed_block->SetCachedValue(block_holder.release(), + block_cache.get(), cache_handle); - UpdateCacheInsertionMetrics(block_type, get_context, charge, + UpdateCacheInsertionMetrics(TBlocklike::kBlockType, get_context, charge, s.IsOkOverwritten(), rep_->ioptions.stats); } else { RecordTick(statistics, BLOCK_CACHE_ADD_FAILURES); @@ -1542,6 +1504,7 @@ InternalIteratorBase* BlockBasedTable::NewIndexIterator( lookup_context); } +// TODO? template <> DataBlockIter* BlockBasedTable::InitBlockIterator( const Rep* rep, Block* block, BlockType block_type, @@ -1551,6 +1514,7 @@ DataBlockIter* BlockBasedTable::InitBlockIterator( rep->ioptions.stats, block_contents_pinned); } +// TODO? template <> IndexBlockIter* BlockBasedTable::InitBlockIterator( const Rep* rep, Block* block, BlockType block_type, @@ -1569,18 +1533,20 @@ IndexBlockIter* BlockBasedTable::InitBlockIterator( // the caller has already read it. In both cases, if ro.fill_cache is true, // it inserts the block into the block cache. template -Status BlockBasedTable::MaybeReadBlockAndLoadToCache( +WithBlocklikeCheck +BlockBasedTable::MaybeReadBlockAndLoadToCache( FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, const BlockHandle& handle, const UncompressionDict& uncompression_dict, const bool wait, const bool for_compaction, - CachableEntry* out_parsed_block, BlockType block_type, - GetContext* get_context, BlockCacheLookupContext* lookup_context, - BlockContents* contents, bool async_read) const { + CachableEntry* out_parsed_block, GetContext* get_context, + BlockCacheLookupContext* lookup_context, BlockContents* contents, + bool async_read) const { assert(out_parsed_block != nullptr); const bool no_io = (ro.read_tier == kBlockCacheTier); - Cache* block_cache = rep_->table_options.block_cache.get(); - Cache* block_cache_compressed = - rep_->table_options.block_cache_compressed.get(); + BlockCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; + CompressedBlockCacheInterface block_cache_compressed{ + rep_->table_options.block_cache_compressed.get()}; // First, try to get the block from the cache // @@ -1589,15 +1555,15 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( CacheKey key_data; Slice key; bool is_cache_hit = false; - if (block_cache != nullptr || block_cache_compressed != nullptr) { + if (block_cache || block_cache_compressed) { // create key for block cache key_data = GetCacheKey(rep_->base_cache_key, handle); key = key_data.AsSlice(); if (!contents) { s = GetDataBlockFromCache(key, block_cache, block_cache_compressed, ro, - out_parsed_block, uncompression_dict, - block_type, wait, get_context); + out_parsed_block, uncompression_dict, wait, + get_context); // Value could still be null at this point, so check the cache handle // and update the read pattern for prefetching if (out_parsed_block->GetValue() || out_parsed_block->GetCacheHandle()) { @@ -1622,8 +1588,8 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( ro.fill_cache) { Statistics* statistics = rep_->ioptions.stats; const bool maybe_compressed = - block_type != BlockType::kFilter && - block_type != BlockType::kCompressionDictionary && + TBlocklike::kBlockType != BlockType::kFilter && + TBlocklike::kBlockType != BlockType::kCompressionDictionary && rep_->blocks_maybe_compressed; const bool do_uncompress = maybe_compressed && !block_cache_compressed; CompressionType contents_comp_type; @@ -1636,7 +1602,8 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( BlockFetcher block_fetcher( rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &tmp_contents, rep_->ioptions, do_uncompress, maybe_compressed, - block_type, uncompression_dict, rep_->persistent_cache_options, + TBlocklike::kBlockType, uncompression_dict, + rep_->persistent_cache_options, GetMemoryAllocator(rep_->table_options), GetMemoryAllocatorForCompressedBlock(rep_->table_options)); @@ -1654,7 +1621,7 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( contents_comp_type = block_fetcher.get_compression_type(); contents = &tmp_contents; if (get_context) { - switch (block_type) { + switch (TBlocklike::kBlockType) { case BlockType::kIndex: ++get_context->get_context_stats_.num_index_read; break; @@ -1676,7 +1643,7 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( s = PutDataBlockToCache( key, block_cache, block_cache_compressed, out_parsed_block, std::move(*contents), contents_comp_type, uncompression_dict, - GetMemoryAllocator(rep_->table_options), block_type, get_context); + GetMemoryAllocator(rep_->table_options), get_context); } } } @@ -1688,13 +1655,13 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( uint64_t nkeys = 0; if (out_parsed_block->GetValue()) { // Approximate the number of keys in the block using restarts. + // FIXME: Should this only apply to data blocks? nkeys = rep_->table_options.block_restart_interval * - BlocklikeTraits::GetNumRestarts( - *out_parsed_block->GetValue()); + GetBlockNumRestarts(*out_parsed_block->GetValue()); usage = out_parsed_block->GetValue()->ApproximateMemoryUsage(); } TraceType trace_block_type = TraceType::kTraceMax; - switch (block_type) { + switch (TBlocklike::kBlockType) { case BlockType::kData: trace_block_type = TraceType::kBlockTraceDataBlock; break; @@ -1750,24 +1717,22 @@ Status BlockBasedTable::MaybeReadBlockAndLoadToCache( return s; } -template -Status BlockBasedTable::RetrieveBlock( +template +WithBlocklikeCheck BlockBasedTable::RetrieveBlock( FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, const BlockHandle& handle, const UncompressionDict& uncompression_dict, - CachableEntry* out_parsed_block, BlockType block_type, - GetContext* get_context, BlockCacheLookupContext* lookup_context, - bool for_compaction, bool use_cache, bool wait_for_cache, - bool async_read) const { + CachableEntry* out_parsed_block, GetContext* get_context, + BlockCacheLookupContext* lookup_context, bool for_compaction, + bool use_cache, bool wait_for_cache, bool async_read) const { assert(out_parsed_block); assert(out_parsed_block->IsEmpty()); Status s; if (use_cache) { - s = MaybeReadBlockAndLoadToCache(prefetch_buffer, ro, handle, - uncompression_dict, wait_for_cache, - for_compaction, out_parsed_block, - block_type, get_context, lookup_context, - /*contents=*/nullptr, async_read); + s = MaybeReadBlockAndLoadToCache( + prefetch_buffer, ro, handle, uncompression_dict, wait_for_cache, + for_compaction, out_parsed_block, get_context, lookup_context, + /*contents=*/nullptr, async_read); if (!s.ok()) { return s; @@ -1788,29 +1753,23 @@ Status BlockBasedTable::RetrieveBlock( } const bool maybe_compressed = - block_type != BlockType::kFilter && - block_type != BlockType::kCompressionDictionary && + TBlocklike::kBlockType != BlockType::kFilter && + TBlocklike::kBlockType != BlockType::kCompressionDictionary && rep_->blocks_maybe_compressed; - const bool do_uncompress = maybe_compressed; std::unique_ptr block; { Histograms histogram = for_compaction ? READ_BLOCK_COMPACTION_MICROS : READ_BLOCK_GET_MICROS; StopWatch sw(rep_->ioptions.clock, rep_->ioptions.stats, histogram); - s = ReadBlockFromFile( + s = ReadAndParseBlockFromFile( rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &block, - rep_->ioptions, do_uncompress, maybe_compressed, block_type, + rep_->ioptions, rep_->create_context, maybe_compressed, uncompression_dict, rep_->persistent_cache_options, - block_type == BlockType::kData - ? rep_->table_options.read_amp_bytes_per_bit - : 0, - GetMemoryAllocator(rep_->table_options), for_compaction, - rep_->blocks_definitely_zstd_compressed, - rep_->table_options.filter_policy.get(), async_read); + GetMemoryAllocator(rep_->table_options), for_compaction, async_read); if (get_context) { - switch (block_type) { + switch (TBlocklike::kBlockType) { case BlockType::kIndex: ++(get_context->get_context_stats_.num_index_read); break; @@ -1834,32 +1793,6 @@ Status BlockBasedTable::RetrieveBlock( return s; } -// Explicitly instantiate templates for each "blocklike" type we use. -// This makes it possible to keep the template definitions in the .cc file. -template Status BlockBasedTable::RetrieveBlock( - FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, - const BlockHandle& handle, const UncompressionDict& uncompression_dict, - CachableEntry* out_parsed_block, - BlockType block_type, GetContext* get_context, - BlockCacheLookupContext* lookup_context, bool for_compaction, - bool use_cache, bool wait_for_cache, bool async_read) const; - -template Status BlockBasedTable::RetrieveBlock( - FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, - const BlockHandle& handle, const UncompressionDict& uncompression_dict, - CachableEntry* out_parsed_block, BlockType block_type, - GetContext* get_context, BlockCacheLookupContext* lookup_context, - bool for_compaction, bool use_cache, bool wait_for_cache, - bool async_read) const; - -template Status BlockBasedTable::RetrieveBlock( - FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, - const BlockHandle& handle, const UncompressionDict& uncompression_dict, - CachableEntry* out_parsed_block, BlockType block_type, - GetContext* get_context, BlockCacheLookupContext* lookup_context, - bool for_compaction, bool use_cache, bool wait_for_cache, - bool async_read) const; - BlockBasedTable::PartitionedIndexIteratorState::PartitionedIndexIteratorState( const BlockBasedTable* table, UnorderedMap>* block_map) diff --git a/table/block_based/block_based_table_reader.h b/table/block_based/block_based_table_reader.h index 89de891c98..55ef76c45f 100644 --- a/table/block_based/block_based_table_reader.h +++ b/table/block_based/block_based_table_reader.h @@ -21,6 +21,7 @@ #include "rocksdb/table_properties.h" #include "table/block_based/block.h" #include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_cache.h" #include "table/block_based/block_type.h" #include "table/block_based/cachable_entry.h" #include "table/block_based/filter_block.h" @@ -315,22 +316,6 @@ class BlockBasedTable : public TableReader { void UpdateCacheMissMetrics(BlockType block_type, GetContext* get_context) const; - Cache::Handle* GetEntryFromCache(const CacheTier& cache_tier, - Cache* block_cache, const Slice& key, - BlockType block_type, const bool wait, - GetContext* get_context, - const Cache::CacheItemHelper* cache_helper, - const Cache::CreateCallback& create_cb, - Cache::Priority priority) const; - - template - Status InsertEntryToCache(const CacheTier& cache_tier, Cache* block_cache, - const Slice& key, - const Cache::CacheItemHelper* cache_helper, - std::unique_ptr&& block_holder, - size_t charge, Cache::Handle** cache_handle, - Cache::Priority priority) const; - // Either Block::NewDataIterator() or Block::NewIndexIterator(). template static TBlockIter* InitBlockIterator(const Rep* rep, Block* block, @@ -348,26 +333,24 @@ class BlockBasedTable : public TableReader { // in uncompressed block cache, also sets cache_handle to reference that // block. template - Status MaybeReadBlockAndLoadToCache( + WithBlocklikeCheck MaybeReadBlockAndLoadToCache( FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, const BlockHandle& handle, const UncompressionDict& uncompression_dict, const bool wait, const bool for_compaction, - CachableEntry* block_entry, BlockType block_type, - GetContext* get_context, BlockCacheLookupContext* lookup_context, - BlockContents* contents, bool async_read) const; + CachableEntry* block_entry, GetContext* get_context, + BlockCacheLookupContext* lookup_context, BlockContents* contents, + bool async_read) const; // Similar to the above, with one crucial difference: it will retrieve the // block from the file even if there are no caches configured (assuming the // read options allow I/O). template - Status RetrieveBlock(FilePrefetchBuffer* prefetch_buffer, - const ReadOptions& ro, const BlockHandle& handle, - const UncompressionDict& uncompression_dict, - CachableEntry* block_entry, - BlockType block_type, GetContext* get_context, - BlockCacheLookupContext* lookup_context, - bool for_compaction, bool use_cache, bool wait_for_cache, - bool async_read) const; + WithBlocklikeCheck RetrieveBlock( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, + const BlockHandle& handle, const UncompressionDict& uncompression_dict, + CachableEntry* block_entry, GetContext* get_context, + BlockCacheLookupContext* lookup_context, bool for_compaction, + bool use_cache, bool wait_for_cache, bool async_read) const; DECLARE_SYNC_AND_ASYNC_CONST( void, RetrieveMultipleBlocks, const ReadOptions& options, @@ -403,13 +386,12 @@ class BlockBasedTable : public TableReader { // @param uncompression_dict Data for presetting the compression library's // dictionary. template - Status GetDataBlockFromCache(const Slice& cache_key, Cache* block_cache, - Cache* block_cache_compressed, - const ReadOptions& read_options, - CachableEntry* block, - const UncompressionDict& uncompression_dict, - BlockType block_type, const bool wait, - GetContext* get_context) const; + WithBlocklikeCheck GetDataBlockFromCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CompressedBlockCacheInterface block_cache_compressed, + const ReadOptions& read_options, CachableEntry* block, + const UncompressionDict& uncompression_dict, const bool wait, + GetContext* get_context) const; // Put a maybe compressed block to the corresponding block caches. // This method will perform decompression against block_contents if needed @@ -422,15 +404,13 @@ class BlockBasedTable : public TableReader { // @param uncompression_dict Data for presetting the compression library's // dictionary. template - Status PutDataBlockToCache(const Slice& cache_key, Cache* block_cache, - Cache* block_cache_compressed, - CachableEntry* cached_block, - BlockContents&& block_contents, - CompressionType block_comp_type, - const UncompressionDict& uncompression_dict, - MemoryAllocator* memory_allocator, - BlockType block_type, - GetContext* get_context) const; + WithBlocklikeCheck PutDataBlockToCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CompressedBlockCacheInterface block_cache_compressed, + CachableEntry* cached_block, BlockContents&& block_contents, + CompressionType block_comp_type, + const UncompressionDict& uncompression_dict, + MemoryAllocator* memory_allocator, GetContext* get_context) const; // Calls (*handle_result)(arg, ...) repeatedly, starting with the entry found // after a call to Seek(key), until handle_result returns false. @@ -599,6 +579,13 @@ struct BlockBasedTable::Rep { std::shared_ptr fragmented_range_dels; + // FIXME + // If true, data blocks in this file are definitely ZSTD compressed. If false + // they might not be. When false we skip creating a ZSTD digested + // uncompression dictionary. Even if we get a false negative, things should + // still work, just not as quickly. + BlockCreateContext create_context; + // If global_seqno is used, all Keys in this file will have the same // seqno with value `global_seqno`. // @@ -617,12 +604,6 @@ struct BlockBasedTable::Rep { // before reading individual blocks enables certain optimizations. bool blocks_maybe_compressed = true; - // If true, data blocks in this file are definitely ZSTD compressed. If false - // they might not be. When false we skip creating a ZSTD digested - // uncompression dictionary. Even if we get a false negative, things should - // still work, just not as quickly. - bool blocks_definitely_zstd_compressed = false; - // These describe how index is encoded. bool index_has_first_key = false; bool index_key_includes_seq = true; diff --git a/table/block_based/block_based_table_reader_impl.h b/table/block_based/block_based_table_reader_impl.h index 0c52973a5d..105a479f36 100644 --- a/table/block_based/block_based_table_reader_impl.h +++ b/table/block_based/block_based_table_reader_impl.h @@ -7,6 +7,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once +#include + +#include "block.h" +#include "block_cache.h" #include "table/block_based/block_based_table_reader.h" #include "table/block_based/reader_common.h" @@ -16,6 +20,25 @@ // are templates. namespace ROCKSDB_NAMESPACE { +namespace { +using IterPlaceholderCacheInterface = + PlaceholderCacheInterface; + +template +struct IterTraits {}; + +template <> +struct IterTraits { + using IterBlocklike = Block_kData; +}; + +template <> +struct IterTraits { + using IterBlocklike = Block_kIndex; +}; + +} // namespace + // Convert an index iterator value (i.e., an encoded BlockHandle) // into an iterator over the contents of the corresponding block. // If input_iter is null, new a iterator @@ -27,6 +50,7 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( BlockCacheLookupContext* lookup_context, FilePrefetchBuffer* prefetch_buffer, bool for_compaction, bool async_read, Status& s) const { + using IterBlocklike = typename IterTraits::IterBlocklike; PERF_TIMER_GUARD(new_table_block_iter_nanos); TBlockIter* iter = input_iter != nullptr ? input_iter : new TBlockIter; @@ -53,14 +77,14 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( const UncompressionDict& dict = uncompression_dict.GetValue() ? *uncompression_dict.GetValue() : UncompressionDict::GetEmptyDict(); - s = RetrieveBlock(prefetch_buffer, ro, handle, dict, &block, block_type, - get_context, lookup_context, for_compaction, - /* use_cache */ true, /* wait_for_cache */ true, - async_read); + s = RetrieveBlock( + prefetch_buffer, ro, handle, dict, &block.As(), + get_context, lookup_context, for_compaction, + /* use_cache */ true, /* wait_for_cache */ true, async_read); } else { s = RetrieveBlock( - prefetch_buffer, ro, handle, UncompressionDict::GetEmptyDict(), &block, - block_type, get_context, lookup_context, for_compaction, + prefetch_buffer, ro, handle, UncompressionDict::GetEmptyDict(), + &block.As(), get_context, lookup_context, for_compaction, /* use_cache */ true, /* wait_for_cache */ true, async_read); } @@ -91,18 +115,20 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( if (!block.IsCached()) { if (!ro.fill_cache) { - Cache* const block_cache = rep_->table_options.block_cache.get(); + IterPlaceholderCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; if (block_cache) { // insert a dummy record to block cache to track the memory usage Cache::Handle* cache_handle = nullptr; - CacheKey key = CacheKey::CreateUniqueForCacheLifetime(block_cache); - s = block_cache->Insert(key.AsSlice(), nullptr, - block.GetValue()->ApproximateMemoryUsage(), - nullptr, &cache_handle); + CacheKey key = + CacheKey::CreateUniqueForCacheLifetime(block_cache.get()); + s = block_cache.Insert(key.AsSlice(), + block.GetValue()->ApproximateMemoryUsage(), + &cache_handle); if (s.ok()) { assert(cache_handle != nullptr); - iter->RegisterCleanup(&ForceReleaseCachedEntry, block_cache, + iter->RegisterCleanup(&ForceReleaseCachedEntry, block_cache.get(), cache_handle); } } @@ -149,18 +175,20 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator(const ReadOptions& ro, if (!block.IsCached()) { if (!ro.fill_cache) { - Cache* const block_cache = rep_->table_options.block_cache.get(); + IterPlaceholderCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; if (block_cache) { // insert a dummy record to block cache to track the memory usage Cache::Handle* cache_handle = nullptr; - CacheKey key = CacheKey::CreateUniqueForCacheLifetime(block_cache); - s = block_cache->Insert(key.AsSlice(), nullptr, - block.GetValue()->ApproximateMemoryUsage(), - nullptr, &cache_handle); + CacheKey key = + CacheKey::CreateUniqueForCacheLifetime(block_cache.get()); + s = block_cache.Insert(key.AsSlice(), + block.GetValue()->ApproximateMemoryUsage(), + &cache_handle); if (s.ok()) { assert(cache_handle != nullptr); - iter->RegisterCleanup(&ForceReleaseCachedEntry, block_cache, + iter->RegisterCleanup(&ForceReleaseCachedEntry, block_cache.get(), cache_handle); } } diff --git a/table/block_based/block_based_table_reader_sync_and_async.h b/table/block_based/block_based_table_reader_sync_and_async.h index 8c7547a2a4..ea75f631d9 100644 --- a/table/block_based/block_based_table_reader_sync_and_async.h +++ b/table/block_based/block_based_table_reader_sync_and_async.h @@ -54,7 +54,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) (*statuses)[idx_in_batch] = RetrieveBlock(nullptr, options, handle, uncompression_dict, - &(*results)[idx_in_batch], BlockType::kData, + &(*results)[idx_in_batch].As(), mget_iter->get_context, &lookup_data_block_context, /* for_compaction */ false, /* use_cache */ true, /* wait_for_cache */ true, /* async_read */ false); @@ -269,7 +269,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) // will avoid looking up the block cache s = MaybeReadBlockAndLoadToCache( nullptr, options, handle, uncompression_dict, /*wait=*/true, - /*for_compaction=*/false, block_entry, BlockType::kData, + /*for_compaction=*/false, &block_entry->As(), mget_iter->get_context, &lookup_data_block_context, &serialized_block, /*async_read=*/false); @@ -441,7 +441,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) ? *uncompression_dict.GetValue() : UncompressionDict::GetEmptyDict(); Status s = RetrieveBlock( - nullptr, ro, handle, dict, &(results.back()), BlockType::kData, + nullptr, ro, handle, dict, &(results.back()).As(), miter->get_context, &lookup_data_block_context, /* for_compaction */ false, /* use_cache */ true, /* wait_for_cache */ false, /* async_read */ false); diff --git a/table/block_based/block_cache.cc b/table/block_based/block_cache.cc new file mode 100644 index 0000000000..86a3918448 --- /dev/null +++ b/table/block_based/block_cache.cc @@ -0,0 +1,96 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/block_cache.h" + +namespace ROCKSDB_NAMESPACE { + +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kData( + std::move(block), table_options->read_amp_bytes_per_bit, statistics)); +} +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kIndex(std::move(block), + /*read_amp_bytes_per_bit*/ 0, statistics)); +} +void BlockCreateContext::Create( + std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kFilterPartitionIndex( + std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics)); +} +void BlockCreateContext::Create( + std::unique_ptr* parsed_out, BlockContents&& block) { + parsed_out->reset(new Block_kRangeDeletion( + std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics)); +} +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kMetaIndex( + std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics)); +} + +void BlockCreateContext::Create( + std::unique_ptr* parsed_out, BlockContents&& block) { + parsed_out->reset(new ParsedFullFilterBlock( + table_options->filter_policy.get(), std::move(block))); +} + +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new UncompressionDict( + block.data, std::move(block.allocation), using_zstd)); +} + +namespace { +// For getting SecondaryCache-compatible helpers from a BlockType. This is +// useful for accessing block cache in untyped contexts, such as for generic +// cache warming in table builder. +constexpr std::array(BlockType::kInvalid) + 1> + kCacheItemFullHelperForBlockType{{ + &BlockCacheInterface::kFullHelper, + &BlockCacheInterface::kFullHelper, + &BlockCacheInterface::kFullHelper, + nullptr, // kProperties + &BlockCacheInterface::kFullHelper, + &BlockCacheInterface::kFullHelper, + nullptr, // kHashIndexPrefixes + nullptr, // kHashIndexMetadata + nullptr, // kMetaIndex (not yet stored in block cache) + &BlockCacheInterface::kFullHelper, + nullptr, // kInvalid + }}; + +// For getting basic helpers from a BlockType (no SecondaryCache support) +constexpr std::array(BlockType::kInvalid) + 1> + kCacheItemBasicHelperForBlockType{{ + &BlockCacheInterface::kBasicHelper, + &BlockCacheInterface::kBasicHelper, + &BlockCacheInterface::kBasicHelper, + nullptr, // kProperties + &BlockCacheInterface::kBasicHelper, + &BlockCacheInterface::kBasicHelper, + nullptr, // kHashIndexPrefixes + nullptr, // kHashIndexMetadata + nullptr, // kMetaIndex (not yet stored in block cache) + &BlockCacheInterface::kBasicHelper, + nullptr, // kInvalid + }}; +} // namespace + +const Cache::CacheItemHelper* GetCacheItemHelper( + BlockType block_type, CacheTier lowest_used_cache_tier) { + if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + return kCacheItemFullHelperForBlockType[static_cast(block_type)]; + } else { + return kCacheItemBasicHelperForBlockType[static_cast(block_type)]; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_cache.h b/table/block_based/block_cache.h new file mode 100644 index 0000000000..8a881595ba --- /dev/null +++ b/table/block_based/block_cache.h @@ -0,0 +1,132 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// Code supporting block cache (Cache) access for block-based table, based on +// the convenient APIs in typed_cache.h + +#pragma once + +#include + +#include "cache/typed_cache.h" +#include "port/lang.h" +#include "table/block_based/block.h" +#include "table/block_based/block_type.h" +#include "table/block_based/parsed_full_filter_block.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +// Metaprogramming wrappers for Block, to give each type a single role when +// used with FullTypedCacheInterface. +// (NOTE: previous attempts to create actual derived classes of Block with +// virtual calls resulted in performance regression) + +class Block_kData : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kDataBlock; + static constexpr BlockType kBlockType = BlockType::kData; +}; + +class Block_kIndex : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kIndexBlock; + static constexpr BlockType kBlockType = BlockType::kIndex; +}; + +class Block_kFilterPartitionIndex : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = + CacheEntryRole::kFilterMetaBlock; + static constexpr BlockType kBlockType = BlockType::kFilterPartitionIndex; +}; + +class Block_kRangeDeletion : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kOtherBlock; + static constexpr BlockType kBlockType = BlockType::kRangeDeletion; +}; + +// Useful for creating the Block even though meta index blocks are not +// yet stored in block cache +class Block_kMetaIndex : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kOtherBlock; + static constexpr BlockType kBlockType = BlockType::kMetaIndex; +}; + +struct BlockCreateContext : public Cache::CreateContext { + BlockCreateContext() {} + BlockCreateContext(const BlockBasedTableOptions* _table_options, + Statistics* _statistics, bool _using_zstd) + : table_options(_table_options), + statistics(_statistics), + using_zstd(_using_zstd) {} + + const BlockBasedTableOptions* table_options = nullptr; + Statistics* statistics = nullptr; + bool using_zstd = false; + + // For TypedCacheInterface + template + inline void Create(std::unique_ptr* parsed_out, + size_t* charge_out, const Slice& data, + MemoryAllocator* alloc) { + Create(parsed_out, + BlockContents(AllocateAndCopyBlock(data, alloc), data.size())); + *charge_out = parsed_out->get()->ApproximateMemoryUsage(); + } + + void Create(std::unique_ptr* parsed_out, BlockContents&& block); + void Create(std::unique_ptr* parsed_out, BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); +}; + +// Convenient cache interface to use with block_cache_compressed +using CompressedBlockCacheInterface = + BasicTypedCacheInterface; + +// Convenient cache interface to use for block_cache, with support for +// SecondaryCache. +template +using BlockCacheInterface = + FullTypedCacheInterface; + +// Shortcut name for cache handles under BlockCacheInterface +template +using BlockCacheTypedHandle = + typename BlockCacheInterface::TypedHandle; + +// Selects the right helper based on BlockType and CacheTier +const Cache::CacheItemHelper* GetCacheItemHelper( + BlockType block_type, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier); + +// For SFINAE check that a type is "blocklike" with a kCacheEntryRole member. +// Can get difficult compiler/linker errors without a good check like this. +template +using WithBlocklikeCheck = std::enable_if_t< + TBlocklike::kCacheEntryRole == CacheEntryRole::kMisc || true, TUse>; + +} // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_like_traits.h b/table/block_based/block_like_traits.h deleted file mode 100644 index d406dbb5dd..0000000000 --- a/table/block_based/block_like_traits.h +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -#pragma once - -#include "cache/cache_entry_roles.h" -#include "port/lang.h" -#include "table/block_based/block.h" -#include "table/block_based/block_type.h" -#include "table/block_based/parsed_full_filter_block.h" -#include "table/format.h" - -namespace ROCKSDB_NAMESPACE { - -template -class BlocklikeTraits; - -template -Cache::CacheItemHelper* GetCacheItemHelperForRole(); - -template -Cache::CreateCallback GetCreateCallback(size_t read_amp_bytes_per_bit, - Statistics* statistics, bool using_zstd, - const FilterPolicy* filter_policy) { - return [read_amp_bytes_per_bit, statistics, using_zstd, filter_policy]( - const void* buf, size_t size, void** out_obj, - size_t* charge) -> Status { - assert(buf != nullptr); - std::unique_ptr buf_data(new char[size]()); - memcpy(buf_data.get(), buf, size); - BlockContents bc = BlockContents(std::move(buf_data), size); - TBlocklike* ucd_ptr = BlocklikeTraits::Create( - std::move(bc), read_amp_bytes_per_bit, statistics, using_zstd, - filter_policy); - *out_obj = reinterpret_cast(ucd_ptr); - *charge = size; - return Status::OK(); - }; -} - -template <> -class BlocklikeTraits { - public: - static ParsedFullFilterBlock* Create(BlockContents&& contents, - size_t /* read_amp_bytes_per_bit */, - Statistics* /* statistics */, - bool /* using_zstd */, - const FilterPolicy* filter_policy) { - return new ParsedFullFilterBlock(filter_policy, std::move(contents)); - } - - static uint32_t GetNumRestarts(const ParsedFullFilterBlock& /* block */) { - return 0; - } - - static size_t SizeCallback(void* obj) { - assert(obj != nullptr); - ParsedFullFilterBlock* ptr = static_cast(obj); - return ptr->GetBlockContentsData().size(); - } - - static Status SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out) { - assert(from_obj != nullptr); - ParsedFullFilterBlock* ptr = static_cast(from_obj); - const char* buf = ptr->GetBlockContentsData().data(); - assert(length == ptr->GetBlockContentsData().size()); - (void)from_offset; - memcpy(out, buf, length); - return Status::OK(); - } - - static Cache::CacheItemHelper* GetCacheItemHelper(BlockType block_type) { - (void)block_type; - assert(block_type == BlockType::kFilter); - return GetCacheItemHelperForRole(); - } -}; - -template <> -class BlocklikeTraits { - public: - static Block* Create(BlockContents&& contents, size_t read_amp_bytes_per_bit, - Statistics* statistics, bool /* using_zstd */, - const FilterPolicy* /* filter_policy */) { - return new Block(std::move(contents), read_amp_bytes_per_bit, statistics); - } - - static uint32_t GetNumRestarts(const Block& block) { - return block.NumRestarts(); - } - - static size_t SizeCallback(void* obj) { - assert(obj != nullptr); - Block* ptr = static_cast(obj); - return ptr->size(); - } - - static Status SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out) { - assert(from_obj != nullptr); - Block* ptr = static_cast(from_obj); - const char* buf = ptr->data(); - assert(length == ptr->size()); - (void)from_offset; - memcpy(out, buf, length); - return Status::OK(); - } - - static Cache::CacheItemHelper* GetCacheItemHelper(BlockType block_type) { - switch (block_type) { - case BlockType::kData: - return GetCacheItemHelperForRole(); - case BlockType::kIndex: - return GetCacheItemHelperForRole(); - case BlockType::kFilterPartitionIndex: - return GetCacheItemHelperForRole(); - default: - // Not a recognized combination - assert(false); - FALLTHROUGH_INTENDED; - case BlockType::kRangeDeletion: - return GetCacheItemHelperForRole(); - } - } -}; - -template <> -class BlocklikeTraits { - public: - static UncompressionDict* Create(BlockContents&& contents, - size_t /* read_amp_bytes_per_bit */, - Statistics* /* statistics */, - bool using_zstd, - const FilterPolicy* /* filter_policy */) { - return new UncompressionDict(contents.data, std::move(contents.allocation), - using_zstd); - } - - static uint32_t GetNumRestarts(const UncompressionDict& /* dict */) { - return 0; - } - - static size_t SizeCallback(void* obj) { - assert(obj != nullptr); - UncompressionDict* ptr = static_cast(obj); - return ptr->slice_.size(); - } - - static Status SaveToCallback(void* from_obj, size_t from_offset, - size_t length, void* out) { - assert(from_obj != nullptr); - UncompressionDict* ptr = static_cast(from_obj); - const char* buf = ptr->slice_.data(); - assert(length == ptr->slice_.size()); - (void)from_offset; - memcpy(out, buf, length); - return Status::OK(); - } - - static Cache::CacheItemHelper* GetCacheItemHelper(BlockType block_type) { - (void)block_type; - assert(block_type == BlockType::kCompressionDictionary); - return GetCacheItemHelperForRole(); - } -}; - -// Get an CacheItemHelper pointer for value type T and role R. -template -Cache::CacheItemHelper* GetCacheItemHelperForRole() { - static Cache::CacheItemHelper cache_helper( - BlocklikeTraits::SizeCallback, BlocklikeTraits::SaveToCallback, - GetCacheEntryDeleterForRole()); - return &cache_helper; -} - -} // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/cachable_entry.h b/table/block_based/cachable_entry.h index ad8acb18d0..464dc8ebaf 100644 --- a/table/block_based/cachable_entry.h +++ b/table/block_based/cachable_entry.h @@ -10,6 +10,7 @@ #pragma once #include +#include #include "port/likely.h" #include "rocksdb/cache.h" @@ -191,6 +192,29 @@ class CachableEntry { return true; } + // Since this class is essentially an elaborate pointer, it's sometimes + // useful to be able to upcast or downcast the base type of the pointer, + // especially when interacting with typed_cache.h. + template + std::enable_if_t || + std::is_base_of_v), + /* Actual return type */ + CachableEntry&> + As() { + CachableEntry* result_ptr = + reinterpret_cast*>(this); + // Ensure no weirdness in template instantiations + assert(static_cast(&this->value_) == + static_cast(&result_ptr->value_)); + assert(&this->cache_handle_ == &result_ptr->cache_handle_); + // This function depends on no arithmetic involved in the pointer + // conversion, which is not statically checkable. + assert(static_cast(this->value_) == + static_cast(result_ptr->value_)); + return *result_ptr; + } + private: void ReleaseResource() noexcept { if (LIKELY(cache_handle_ != nullptr)) { @@ -223,6 +247,10 @@ class CachableEntry { } private: + // Have to be your own best friend + template + friend class CachableEntry; + T* value_ = nullptr; Cache* cache_ = nullptr; Cache::Handle* cache_handle_ = nullptr; diff --git a/table/block_based/filter_block_reader_common.cc b/table/block_based/filter_block_reader_common.cc index 7dc49e83e7..838fb5296a 100644 --- a/table/block_based/filter_block_reader_common.cc +++ b/table/block_based/filter_block_reader_common.cc @@ -6,6 +6,7 @@ #include "table/block_based/filter_block_reader_common.h" +#include "block_cache.h" #include "monitoring/perf_context_imp.h" #include "table/block_based/block_based_table_reader.h" #include "table/block_based/parsed_full_filter_block.h" @@ -17,7 +18,7 @@ Status FilterBlockReaderCommon::ReadFilterBlock( const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, const ReadOptions& read_options, bool use_cache, GetContext* get_context, BlockCacheLookupContext* lookup_context, - CachableEntry* filter_block, BlockType block_type) { + CachableEntry* filter_block) { PERF_TIMER_GUARD(read_filter_block_nanos); assert(table); @@ -30,7 +31,7 @@ Status FilterBlockReaderCommon::ReadFilterBlock( const Status s = table->RetrieveBlock(prefetch_buffer, read_options, rep->filter_handle, UncompressionDict::GetEmptyDict(), filter_block, - block_type, get_context, lookup_context, + get_context, lookup_context, /* for_compaction */ false, use_cache, /* wait_for_cache */ true, /* async_read */ false); @@ -68,7 +69,7 @@ template Status FilterBlockReaderCommon::GetOrReadFilterBlock( bool no_io, GetContext* get_context, BlockCacheLookupContext* lookup_context, - CachableEntry* filter_block, BlockType block_type, + CachableEntry* filter_block, Env::IOPriority rate_limiter_priority) const { assert(filter_block); @@ -85,7 +86,7 @@ Status FilterBlockReaderCommon::GetOrReadFilterBlock( return ReadFilterBlock(table_, nullptr /* prefetch_buffer */, read_options, cache_filter_blocks(), get_context, lookup_context, - filter_block, block_type); + filter_block); } template @@ -158,7 +159,7 @@ bool FilterBlockReaderCommon::IsFilterCompatible( // Explicitly instantiate templates for both "blocklike" types we use. // This makes it possible to keep the template definitions in the .cc file. -template class FilterBlockReaderCommon; +template class FilterBlockReaderCommon; template class FilterBlockReaderCommon; } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/filter_block_reader_common.h b/table/block_based/filter_block_reader_common.h index ca07f50508..5c2fbdcea7 100644 --- a/table/block_based/filter_block_reader_common.h +++ b/table/block_based/filter_block_reader_common.h @@ -8,7 +8,6 @@ #include -#include "block_type.h" #include "table/block_based/cachable_entry.h" #include "table/block_based/filter_block.h" @@ -49,8 +48,7 @@ class FilterBlockReaderCommon : public FilterBlockReader { const ReadOptions& read_options, bool use_cache, GetContext* get_context, BlockCacheLookupContext* lookup_context, - CachableEntry* filter_block, - BlockType block_type); + CachableEntry* filter_block); const BlockBasedTable* table() const { return table_; } const SliceTransform* table_prefix_extractor() const; @@ -60,7 +58,6 @@ class FilterBlockReaderCommon : public FilterBlockReader { Status GetOrReadFilterBlock(bool no_io, GetContext* get_context, BlockCacheLookupContext* lookup_context, CachableEntry* filter_block, - BlockType block_type, Env::IOPriority rate_limiter_priority) const; size_t ApproximateFilterBlockMemoryUsage() const; diff --git a/table/block_based/full_filter_block.cc b/table/block_based/full_filter_block.cc index 62b7a9eca0..a7680e494d 100644 --- a/table/block_based/full_filter_block.cc +++ b/table/block_based/full_filter_block.cc @@ -147,7 +147,7 @@ std::unique_ptr FullFilterBlockReader::Create( if (prefetch || !use_cache) { const Status s = ReadFilterBlock(table, prefetch_buffer, ro, use_cache, nullptr /* get_context */, lookup_context, - &filter_block, BlockType::kFilter); + &filter_block); if (!s.ok()) { IGNORE_STATUS_IF_ERROR(s); return std::unique_ptr(); @@ -177,9 +177,8 @@ bool FullFilterBlockReader::MayMatch( Env::IOPriority rate_limiter_priority) const { CachableEntry filter_block; - const Status s = - GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block, - BlockType::kFilter, rate_limiter_priority); + const Status s = GetOrReadFilterBlock(no_io, get_context, lookup_context, + &filter_block, rate_limiter_priority); if (!s.ok()) { IGNORE_STATUS_IF_ERROR(s); return true; @@ -228,9 +227,9 @@ void FullFilterBlockReader::MayMatch( Env::IOPriority rate_limiter_priority) const { CachableEntry filter_block; - const Status s = GetOrReadFilterBlock( - no_io, range->begin()->get_context, lookup_context, &filter_block, - BlockType::kFilter, rate_limiter_priority); + const Status s = + GetOrReadFilterBlock(no_io, range->begin()->get_context, lookup_context, + &filter_block, rate_limiter_priority); if (!s.ok()) { IGNORE_STATUS_IF_ERROR(s); return; diff --git a/table/block_based/index_reader_common.cc b/table/block_based/index_reader_common.cc index 6584586c9b..46c276e6be 100644 --- a/table/block_based/index_reader_common.cc +++ b/table/block_based/index_reader_common.cc @@ -8,6 +8,8 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "table/block_based/index_reader_common.h" +#include "block_cache.h" + namespace ROCKSDB_NAMESPACE { Status BlockBasedTable::IndexReaderCommon::ReadIndexBlock( const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, @@ -25,7 +27,7 @@ Status BlockBasedTable::IndexReaderCommon::ReadIndexBlock( const Status s = table->RetrieveBlock( prefetch_buffer, read_options, rep->footer.index_handle(), - UncompressionDict::GetEmptyDict(), index_block, BlockType::kIndex, + UncompressionDict::GetEmptyDict(), &index_block->As(), get_context, lookup_context, /* for_compaction */ false, use_cache, /* wait_for_cache */ true, /* async_read */ false); diff --git a/table/block_based/parsed_full_filter_block.h b/table/block_based/parsed_full_filter_block.h index 95d7b52087..8d81868d1b 100644 --- a/table/block_based/parsed_full_filter_block.h +++ b/table/block_based/parsed_full_filter_block.h @@ -7,6 +7,7 @@ #include +#include "table/block_based/block_type.h" #include "table/format.h" namespace ROCKSDB_NAMESPACE { @@ -32,7 +33,11 @@ class ParsedFullFilterBlock { bool own_bytes() const { return block_contents_.own_bytes(); } - const Slice GetBlockContentsData() const { return block_contents_.data; } + // For TypedCacheInterface + const Slice& ContentSlice() const { return block_contents_.data; } + static constexpr CacheEntryRole kCacheEntryRole = + CacheEntryRole::kFilterBlock; + static constexpr BlockType kBlockType = BlockType::kFilter; private: BlockContents block_contents_; diff --git a/table/block_based/partitioned_filter_block.cc b/table/block_based/partitioned_filter_block.cc index af30925b73..092446f022 100644 --- a/table/block_based/partitioned_filter_block.cc +++ b/table/block_based/partitioned_filter_block.cc @@ -7,6 +7,7 @@ #include +#include "block_cache.h" #include "block_type.h" #include "file/random_access_file_reader.h" #include "logging/logging.h" @@ -185,7 +186,8 @@ Slice PartitionedFilterBlockBuilder::Finish( } PartitionedFilterBlockReader::PartitionedFilterBlockReader( - const BlockBasedTable* t, CachableEntry&& filter_block) + const BlockBasedTable* t, + CachableEntry&& filter_block) : FilterBlockReaderCommon(t, std::move(filter_block)) {} std::unique_ptr PartitionedFilterBlockReader::Create( @@ -196,11 +198,11 @@ std::unique_ptr PartitionedFilterBlockReader::Create( assert(table->get_rep()); assert(!pin || prefetch); - CachableEntry filter_block; + CachableEntry filter_block; if (prefetch || !use_cache) { - const Status s = ReadFilterBlock( - table, prefetch_buffer, ro, use_cache, nullptr /* get_context */, - lookup_context, &filter_block, BlockType::kFilterPartitionIndex); + const Status s = ReadFilterBlock(table, prefetch_buffer, ro, use_cache, + nullptr /* get_context */, lookup_context, + &filter_block); if (!s.ok()) { IGNORE_STATUS_IF_ERROR(s); return std::unique_ptr(); @@ -260,7 +262,8 @@ void PartitionedFilterBlockReader::PrefixesMayMatch( } BlockHandle PartitionedFilterBlockReader::GetFilterPartitionHandle( - const CachableEntry& filter_block, const Slice& entry) const { + const CachableEntry& filter_block, + const Slice& entry) const { IndexBlockIter iter; const InternalKeyComparator* const comparator = internal_comparator(); Statistics* kNullStats = nullptr; @@ -313,7 +316,7 @@ Status PartitionedFilterBlockReader::GetFilterPartitionBlock( const Status s = table()->RetrieveBlock(prefetch_buffer, read_options, fltr_blk_handle, UncompressionDict::GetEmptyDict(), filter_block, - BlockType::kFilter, get_context, lookup_context, + get_context, lookup_context, /* for_compaction */ false, /* use_cache */ true, /* wait_for_cache */ true, /* async_read */ false); @@ -325,10 +328,9 @@ bool PartitionedFilterBlockReader::MayMatch( GetContext* get_context, BlockCacheLookupContext* lookup_context, Env::IOPriority rate_limiter_priority, FilterFunction filter_function) const { - CachableEntry filter_block; - Status s = GetOrReadFilterBlock( - no_io, get_context, lookup_context, &filter_block, - BlockType::kFilterPartitionIndex, rate_limiter_priority); + CachableEntry filter_block; + Status s = GetOrReadFilterBlock(no_io, get_context, lookup_context, + &filter_block, rate_limiter_priority); if (UNLIKELY(!s.ok())) { IGNORE_STATUS_IF_ERROR(s); return true; @@ -364,10 +366,10 @@ void PartitionedFilterBlockReader::MayMatch( BlockCacheLookupContext* lookup_context, Env::IOPriority rate_limiter_priority, FilterManyFunction filter_function) const { - CachableEntry filter_block; - Status s = GetOrReadFilterBlock( - no_io, range->begin()->get_context, lookup_context, &filter_block, - BlockType::kFilterPartitionIndex, rate_limiter_priority); + CachableEntry filter_block; + Status s = + GetOrReadFilterBlock(no_io, range->begin()->get_context, lookup_context, + &filter_block, rate_limiter_priority); if (UNLIKELY(!s.ok())) { IGNORE_STATUS_IF_ERROR(s); return; // Any/all may match @@ -455,11 +457,10 @@ Status PartitionedFilterBlockReader::CacheDependencies(const ReadOptions& ro, BlockCacheLookupContext lookup_context{TableReaderCaller::kPrefetch}; - CachableEntry filter_block; + CachableEntry filter_block; Status s = GetOrReadFilterBlock(false /* no_io */, nullptr /* get_context */, &lookup_context, &filter_block, - BlockType::kFilterPartitionIndex, ro.rate_limiter_priority); if (!s.ok()) { ROCKS_LOG_ERROR(rep->ioptions.logger, @@ -517,7 +518,7 @@ Status PartitionedFilterBlockReader::CacheDependencies(const ReadOptions& ro, // filter blocks s = table()->MaybeReadBlockAndLoadToCache( prefetch_buffer.get(), ro, handle, UncompressionDict::GetEmptyDict(), - /* wait */ true, /* for_compaction */ false, &block, BlockType::kFilter, + /* wait */ true, /* for_compaction */ false, &block, nullptr /* get_context */, &lookup_context, nullptr /* contents */, false); if (!s.ok()) { diff --git a/table/block_based/partitioned_filter_block.h b/table/block_based/partitioned_filter_block.h index 955b507396..e810c01eeb 100644 --- a/table/block_based/partitioned_filter_block.h +++ b/table/block_based/partitioned_filter_block.h @@ -10,6 +10,7 @@ #include #include +#include "block_cache.h" #include "rocksdb/options.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" @@ -99,10 +100,12 @@ class PartitionedFilterBlockBuilder : public FullFilterBlockBuilder { BlockHandle last_encoded_handle_; }; -class PartitionedFilterBlockReader : public FilterBlockReaderCommon { +class PartitionedFilterBlockReader + : public FilterBlockReaderCommon { public: - PartitionedFilterBlockReader(const BlockBasedTable* t, - CachableEntry&& filter_block); + PartitionedFilterBlockReader( + const BlockBasedTable* t, + CachableEntry&& filter_block); static std::unique_ptr Create( const BlockBasedTable* table, const ReadOptions& ro, @@ -131,8 +134,9 @@ class PartitionedFilterBlockReader : public FilterBlockReaderCommon { size_t ApproximateMemoryUsage() const override; private: - BlockHandle GetFilterPartitionHandle(const CachableEntry& filter_block, - const Slice& entry) const; + BlockHandle GetFilterPartitionHandle( + const CachableEntry& filter_block, + const Slice& entry) const; Status GetFilterPartitionBlock( FilePrefetchBuffer* prefetch_buffer, const BlockHandle& handle, bool no_io, GetContext* get_context, diff --git a/table/block_based/partitioned_filter_block_test.cc b/table/block_based/partitioned_filter_block_test.cc index 0ce50d2bc5..59445c45e0 100644 --- a/table/block_based/partitioned_filter_block_test.cc +++ b/table/block_based/partitioned_filter_block_test.cc @@ -7,6 +7,7 @@ #include +#include "block_cache.h" #include "index_builder.h" #include "rocksdb/filter_policy.h" #include "table/block_based/block_based_table_reader.h" @@ -35,7 +36,8 @@ class MyPartitionedFilterBlockReader : public PartitionedFilterBlockReader { public: MyPartitionedFilterBlockReader(BlockBasedTable* t, CachableEntry&& filter_block) - : PartitionedFilterBlockReader(t, std::move(filter_block)) { + : PartitionedFilterBlockReader( + t, std::move(filter_block.As())) { for (const auto& pair : blooms) { const uint64_t offset = pair.first; const std::string& bloom = pair.second; diff --git a/table/block_based/partitioned_index_reader.cc b/table/block_based/partitioned_index_reader.cc index 017ea4a3a2..705223c90a 100644 --- a/table/block_based/partitioned_index_reader.cc +++ b/table/block_based/partitioned_index_reader.cc @@ -8,6 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "table/block_based/partitioned_index_reader.h" +#include "block_cache.h" #include "file/random_access_file_reader.h" #include "table/block_based/block_based_table_reader.h" #include "table/block_based/partitioned_index_iterator.h" @@ -186,7 +187,7 @@ Status PartitionIndexReader::CacheDependencies(const ReadOptions& ro, // filter blocks Status s = table()->MaybeReadBlockAndLoadToCache( prefetch_buffer.get(), ro, handle, UncompressionDict::GetEmptyDict(), - /*wait=*/true, /*for_compaction=*/false, &block, BlockType::kIndex, + /*wait=*/true, /*for_compaction=*/false, &block.As(), /*get_context=*/nullptr, &lookup_context, /*contents=*/nullptr, /*async_read=*/false); diff --git a/table/block_based/uncompression_dict_reader.cc b/table/block_based/uncompression_dict_reader.cc index dc9a47ec73..7b0b7c9435 100644 --- a/table/block_based/uncompression_dict_reader.cc +++ b/table/block_based/uncompression_dict_reader.cc @@ -60,8 +60,8 @@ Status UncompressionDictReader::ReadUncompressionDictionary( const Status s = table->RetrieveBlock( prefetch_buffer, read_options, rep->compression_dict_handle, - UncompressionDict::GetEmptyDict(), uncompression_dict, - BlockType::kCompressionDictionary, get_context, lookup_context, + UncompressionDict::GetEmptyDict(), uncompression_dict, get_context, + lookup_context, /* for_compaction */ false, use_cache, /* wait_for_cache */ true, /* async_read */ false); diff --git a/table/format.h b/table/format.h index 23e6b891cb..71d3706c42 100644 --- a/table/format.h +++ b/table/format.h @@ -276,7 +276,7 @@ uint32_t ComputeBuiltinChecksumWithLastByte(ChecksumType type, const char* data, // decompression function. // * "Parsed block" - an in-memory form of a block in block cache, as it is // used by the table reader. Different C++ types are used depending on the -// block type (see block_like_traits.h). Only trivially parsable block types +// block type (see block_cache.h). Only trivially parsable block types // use BlockContents as the parsed form. // struct BlockContents { diff --git a/util/compression.h b/util/compression.h index 0d4febcfb2..2185d52134 100644 --- a/util/compression.h +++ b/util/compression.h @@ -23,6 +23,7 @@ #include "memory/memory_allocator.h" #include "rocksdb/options.h" #include "rocksdb/table.h" +#include "table/block_based/block_type.h" #include "test_util/sync_point.h" #include "util/coding.h" #include "util/compression_context_cache.h" @@ -321,6 +322,11 @@ struct UncompressionDict { const Slice& GetRawDict() const { return slice_; } + // For TypedCacheInterface + const Slice& ContentSlice() const { return slice_; } + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kOtherBlock; + static constexpr BlockType kBlockType = BlockType::kCompressionDictionary; + #ifdef ROCKSDB_ZSTD_DDICT const ZSTD_DDict* GetDigestedZstdDDict() const { return zstd_ddict_; } #endif // ROCKSDB_ZSTD_DDICT diff --git a/utilities/cache_dump_load_impl.cc b/utilities/cache_dump_load_impl.cc index 2b9f2a29d2..b5e21291b3 100644 --- a/utilities/cache_dump_load_impl.cc +++ b/utilities/cache_dump_load_impl.cc @@ -67,8 +67,7 @@ IOStatus CacheDumperImpl::DumpCacheEntriesToWriter() { return IOStatus::InvalidArgument("System clock is null"); } clock_ = options_.clock; - // We copy the Cache Deleter Role Map as its member. - role_map_ = CopyCacheDeleterRoleMap(); + // Set the sequence number sequence_num_ = 0; @@ -80,7 +79,8 @@ IOStatus CacheDumperImpl::DumpCacheEntriesToWriter() { // Then, we iterate the block cache and dump out the blocks that are not // filtered out. - cache_->ApplyToAllEntries(DumpOneBlockCallBack(), {}); + std::string buf; + cache_->ApplyToAllEntries(DumpOneBlockCallBack(buf), {}); // Finally, write the footer io_s = WriteFooter(); @@ -105,77 +105,57 @@ bool CacheDumperImpl::ShouldFilterOut(const Slice& key) { // This is the callback function which will be applied to // Cache::ApplyToAllEntries. In this callback function, we will get the block // type, decide if the block needs to be dumped based on the filter, and write -// the block through the provided writer. -std::function -CacheDumperImpl::DumpOneBlockCallBack() { - return [&](const Slice& key, void* value, size_t /*charge*/, - Cache::DeleterFn deleter) { - // Step 1: get the type of the block from role_map_ - auto e = role_map_.find(deleter); - CacheEntryRole role; +// the block through the provided writer. `buf` is passed in for efficiennt +// reuse. +std::function +CacheDumperImpl::DumpOneBlockCallBack(std::string& buf) { + return [&](const Slice& key, Cache::ObjectPtr value, size_t /*charge*/, + const Cache::CacheItemHelper* helper) { + if (helper == nullptr || helper->size_cb == nullptr || + helper->saveto_cb == nullptr) { + // Not compatible with dumping. Skip this entry. + return; + } + + CacheEntryRole role = helper->role; CacheDumpUnitType type = CacheDumpUnitType::kBlockTypeMax; - if (e == role_map_.end()) { - role = CacheEntryRole::kMisc; - } else { - role = e->second; - } - bool filter_out = false; - // Step 2: based on the key prefix, check if the block should be filter out. - if (ShouldFilterOut(key)) { - filter_out = true; - } - - // Step 3: based on the block type, get the block raw pointer and length. - const char* block_start = nullptr; - size_t block_len = 0; switch (role) { case CacheEntryRole::kDataBlock: type = CacheDumpUnitType::kData; - block_start = (static_cast(value))->data(); - block_len = (static_cast(value))->size(); break; case CacheEntryRole::kFilterBlock: type = CacheDumpUnitType::kFilter; - block_start = (static_cast(value)) - ->GetBlockContentsData() - .data(); - block_len = (static_cast(value)) - ->GetBlockContentsData() - .size(); break; case CacheEntryRole::kFilterMetaBlock: type = CacheDumpUnitType::kFilterMetaBlock; - block_start = (static_cast(value))->data(); - block_len = (static_cast(value))->size(); break; case CacheEntryRole::kIndexBlock: type = CacheDumpUnitType::kIndex; - block_start = (static_cast(value))->data(); - block_len = (static_cast(value))->size(); - break; - case CacheEntryRole::kDeprecatedFilterBlock: - // Obsolete - filter_out = true; - break; - case CacheEntryRole::kMisc: - filter_out = true; - break; - case CacheEntryRole::kOtherBlock: - filter_out = true; - break; - case CacheEntryRole::kWriteBuffer: - filter_out = true; break; default: - filter_out = true; + // Filter out other entries + // FIXME? Do we need the CacheDumpUnitTypes? UncompressionDict? + return; } - // Step 4: if the block should not be filter out, write the block to the - // CacheDumpWriter - if (!filter_out && block_start != nullptr) { - WriteBlock(type, key, Slice(block_start, block_len)) - .PermitUncheckedError(); + // based on the key prefix, check if the block should be filter out. + if (ShouldFilterOut(key)) { + return; + } + + assert(type != CacheDumpUnitType::kBlockTypeMax); + + // Use cache item helper to get persistable data + // FIXME: reduce copying + size_t len = helper->size_cb(value); + buf.assign(len, '\0'); + Status s = helper->saveto_cb(value, /*start*/ 0, len, buf.data()); + + if (s.ok()) { + // Write it out + WriteBlock(type, key, buf).PermitUncheckedError(); } }; } @@ -264,8 +244,6 @@ IOStatus CacheDumpedLoaderImpl::RestoreCacheEntriesToSecondaryCache() { if (reader_ == nullptr) { return IOStatus::InvalidArgument("CacheDumpReader is null"); } - // we copy the Cache Deleter Role Map as its member. - role_map_ = CopyCacheDeleterRoleMap(); // Step 2: read the header // TODO: we need to check the cache dump format version and RocksDB version diff --git a/utilities/cache_dump_load_impl.h b/utilities/cache_dump_load_impl.h index 9ca1ff45aa..ad637b00dd 100644 --- a/utilities/cache_dump_load_impl.h +++ b/utilities/cache_dump_load_impl.h @@ -12,11 +12,11 @@ #include "file/writable_file_writer.h" #include "rocksdb/utilities/cache_dump_load.h" #include "table/block_based/block.h" -#include "table/block_based/block_like_traits.h" #include "table/block_based/block_type.h" #include "table/block_based/cachable_entry.h" #include "table/block_based/parsed_full_filter_block.h" #include "table/block_based/reader_common.h" +#include "util/hash_containers.h" namespace ROCKSDB_NAMESPACE { @@ -108,13 +108,13 @@ class CacheDumperImpl : public CacheDumper { IOStatus WriteHeader(); IOStatus WriteFooter(); bool ShouldFilterOut(const Slice& key); - std::function - DumpOneBlockCallBack(); + std::function + DumpOneBlockCallBack(std::string& buf); CacheDumpOptions options_; std::shared_ptr cache_; std::unique_ptr writer_; - UnorderedMap role_map_; SystemClock* clock_; uint32_t sequence_num_; // The cache key prefix filter. Currently, we use db_session_id as the prefix, @@ -146,7 +146,6 @@ class CacheDumpedLoaderImpl : public CacheDumpedLoader { CacheDumpOptions options_; std::shared_ptr secondary_cache_; std::unique_ptr reader_; - UnorderedMap role_map_; }; // The default implementation of CacheDumpWriter. We write the blocks to a file diff --git a/utilities/fault_injection_secondary_cache.cc b/utilities/fault_injection_secondary_cache.cc index 2758c2a193..d24e92f06f 100644 --- a/utilities/fault_injection_secondary_cache.cc +++ b/utilities/fault_injection_secondary_cache.cc @@ -36,7 +36,9 @@ void FaultInjectionSecondaryCache::ResultHandle::Wait() { UpdateHandleValue(this); } -void* FaultInjectionSecondaryCache::ResultHandle::Value() { return value_; } +Cache::ObjectPtr FaultInjectionSecondaryCache::ResultHandle::Value() { + return value_; +} size_t FaultInjectionSecondaryCache::ResultHandle::Size() { return size_; } @@ -75,7 +77,8 @@ FaultInjectionSecondaryCache::GetErrorContext() { } Status FaultInjectionSecondaryCache::Insert( - const Slice& key, void* value, const Cache::CacheItemHelper* helper) { + const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper) { ErrorContext* ctx = GetErrorContext(); if (ctx->rand.OneIn(prob_)) { return Status::IOError(); @@ -86,7 +89,8 @@ Status FaultInjectionSecondaryCache::Insert( std::unique_ptr FaultInjectionSecondaryCache::Lookup(const Slice& key, - const Cache::CreateCallback& create_cb, + const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, bool& is_in_sec_cache) { ErrorContext* ctx = GetErrorContext(); @@ -94,11 +98,12 @@ FaultInjectionSecondaryCache::Lookup(const Slice& key, if (ctx->rand.OneIn(prob_)) { return nullptr; } else { - return base_->Lookup(key, create_cb, wait, advise_erase, is_in_sec_cache); + return base_->Lookup(key, helper, create_context, wait, advise_erase, + is_in_sec_cache); } } else { - std::unique_ptr hdl = - base_->Lookup(key, create_cb, wait, advise_erase, is_in_sec_cache); + std::unique_ptr hdl = base_->Lookup( + key, helper, create_context, wait, advise_erase, is_in_sec_cache); if (wait && ctx->rand.OneIn(prob_)) { hdl.reset(); } diff --git a/utilities/fault_injection_secondary_cache.h b/utilities/fault_injection_secondary_cache.h index 5321df626c..47585e30e0 100644 --- a/utilities/fault_injection_secondary_cache.h +++ b/utilities/fault_injection_secondary_cache.h @@ -31,12 +31,13 @@ class FaultInjectionSecondaryCache : public SecondaryCache { const char* Name() const override { return "FaultInjectionSecondaryCache"; } - Status Insert(const Slice& key, void* value, + Status Insert(const Slice& key, Cache::ObjectPtr value, const Cache::CacheItemHelper* helper) override; std::unique_ptr Lookup( - const Slice& key, const Cache::CreateCallback& create_cb, bool wait, - bool advise_erase, bool& is_in_sec_cache) override; + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, + bool& is_in_sec_cache) override; bool SupportForceErase() const override { return base_->SupportForceErase(); } @@ -69,7 +70,7 @@ class FaultInjectionSecondaryCache : public SecondaryCache { void Wait() override; - void* Value() override; + Cache::ObjectPtr Value() override; size_t Size() override; @@ -81,7 +82,7 @@ class FaultInjectionSecondaryCache : public SecondaryCache { FaultInjectionSecondaryCache* cache_; std::unique_ptr base_; - void* value_; + Cache::ObjectPtr value_; size_t size_; }; diff --git a/utilities/memory_allocators.h b/utilities/memory_allocators.h index c9e77a5b7d..bdc2e13a94 100644 --- a/utilities/memory_allocators.h +++ b/utilities/memory_allocators.h @@ -6,7 +6,6 @@ #pragma once #include - #include "rocksdb/memory_allocator.h" namespace ROCKSDB_NAMESPACE { diff --git a/utilities/simulator_cache/cache_simulator.cc b/utilities/simulator_cache/cache_simulator.cc index dc419e51ab..edb75d545c 100644 --- a/utilities/simulator_cache/cache_simulator.cc +++ b/utilities/simulator_cache/cache_simulator.cc @@ -26,8 +26,8 @@ bool GhostCache::Admit(const Slice& lookup_key) { return true; } // TODO: Should we check for errors here? - auto s = sim_cache_->Insert(lookup_key, /*value=*/nullptr, lookup_key.size(), - /*deleter=*/nullptr); + auto s = sim_cache_->Insert(lookup_key, /*obj=*/nullptr, + &kNoopCacheItemHelper, lookup_key.size()); s.PermitUncheckedError(); return false; } @@ -51,9 +51,8 @@ void CacheSimulator::Access(const BlockCacheTraceRecord& access) { } else { if (!access.no_insert && admit && access.block_size > 0) { // Ignore errors on insert - auto s = sim_cache_->Insert(access.block_key, /*value=*/nullptr, - access.block_size, - /*deleter=*/nullptr); + auto s = sim_cache_->Insert(access.block_key, /*obj=*/nullptr, + &kNoopCacheItemHelper, access.block_size); s.PermitUncheckedError(); } } @@ -109,8 +108,8 @@ void PrioritizedCacheSimulator::AccessKVPair( *is_cache_miss = false; } else if (!no_insert && *admitted && value_size > 0) { // TODO: Should we check for an error here? - auto s = sim_cache_->Insert(key, /*value=*/nullptr, value_size, - /*deleter=*/nullptr, + auto s = sim_cache_->Insert(key, /*obj=*/nullptr, &kNoopCacheItemHelper, + value_size, /*handle=*/nullptr, priority); s.PermitUncheckedError(); } @@ -188,10 +187,10 @@ void HybridRowBlockCacheSimulator::Access(const BlockCacheTraceRecord& access) { /*update_metrics=*/true); if (access.referenced_data_size > 0 && inserted == InsertResult::ADMITTED) { // TODO: Should we check for an error here? - auto s = sim_cache_->Insert(row_key, /*value=*/nullptr, - access.referenced_data_size, - /*deleter=*/nullptr, - /*handle=*/nullptr, Cache::Priority::HIGH); + auto s = + sim_cache_->Insert(row_key, /*obj=*/nullptr, &kNoopCacheItemHelper, + access.referenced_data_size, + /*handle=*/nullptr, Cache::Priority::HIGH); s.PermitUncheckedError(); status.row_key_status[row_key] = InsertResult::INSERTED; } diff --git a/utilities/simulator_cache/sim_cache.cc b/utilities/simulator_cache/sim_cache.cc index a883b52e78..0f0c098710 100644 --- a/utilities/simulator_cache/sim_cache.cc +++ b/utilities/simulator_cache/sim_cache.cc @@ -165,8 +165,8 @@ class SimCacheImpl : public SimCache { } using Cache::Insert; - Status Insert(const Slice& key, void* value, size_t charge, - void (*deleter)(const Slice& key, void* value), Handle** handle, + Status Insert(const Slice& key, Cache::ObjectPtr value, + const CacheItemHelper* helper, size_t charge, Handle** handle, Priority priority) override { // The handle and value passed in are for real cache, so we pass nullptr // to key_only_cache_ for both instead. Also, the deleter function pointer @@ -176,9 +176,8 @@ class SimCacheImpl : public SimCache { Handle* h = key_only_cache_->Lookup(key); if (h == nullptr) { // TODO: Check for error here? - auto s = key_only_cache_->Insert( - key, nullptr, charge, [](const Slice& /*k*/, void* /*v*/) {}, nullptr, - priority); + auto s = key_only_cache_->Insert(key, nullptr, &kNoopCacheItemHelper, + charge, nullptr, priority); s.PermitUncheckedError(); } else { key_only_cache_->Release(h); @@ -188,26 +187,18 @@ class SimCacheImpl : public SimCache { if (!cache_) { return Status::OK(); } - return cache_->Insert(key, value, charge, deleter, handle, priority); + return cache_->Insert(key, value, helper, charge, handle, priority); } - using Cache::Lookup; - Handle* Lookup(const Slice& key, Statistics* stats) override { - Handle* h = key_only_cache_->Lookup(key); - if (h != nullptr) { - key_only_cache_->Release(h); - inc_hit_counter(); - RecordTick(stats, SIM_BLOCK_CACHE_HIT); - } else { - inc_miss_counter(); - RecordTick(stats, SIM_BLOCK_CACHE_MISS); - } - - cache_activity_logger_.ReportLookup(key); + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, bool wait = true, + Statistics* stats = nullptr) override { + HandleLookup(key, stats); if (!cache_) { return nullptr; } - return cache_->Lookup(key, stats); + return cache_->Lookup(key, helper, create_context, priority, wait, stats); } bool Ref(Handle* handle) override { return cache_->Ref(handle); } @@ -222,7 +213,9 @@ class SimCacheImpl : public SimCache { key_only_cache_->Erase(key); } - void* Value(Handle* handle) override { return cache_->Value(handle); } + Cache::ObjectPtr Value(Handle* handle) override { + return cache_->Value(handle); + } uint64_t NewId() override { return cache_->NewId(); } @@ -242,8 +235,8 @@ class SimCacheImpl : public SimCache { return cache_->GetCharge(handle); } - DeleterFn GetDeleter(Handle* handle) const override { - return cache_->GetDeleter(handle); + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override { + return cache_->GetCacheItemHelper(handle); } size_t GetPinnedUsage() const override { return cache_->GetPinnedUsage(); } @@ -253,15 +246,9 @@ class SimCacheImpl : public SimCache { key_only_cache_->DisownData(); } - void ApplyToAllCacheEntries(void (*callback)(void*, size_t), - bool thread_safe) override { - // only apply to _cache since key_only_cache doesn't hold value - cache_->ApplyToAllCacheEntries(callback, thread_safe); - } - void ApplyToAllEntries( - const std::function& callback, + const std::function& callback, const ApplyToAllEntriesOptions& opts) override { cache_->ApplyToAllEntries(callback, opts); } @@ -338,6 +325,19 @@ class SimCacheImpl : public SimCache { miss_times_.fetch_add(1, std::memory_order_relaxed); } void inc_hit_counter() { hit_times_.fetch_add(1, std::memory_order_relaxed); } + + void HandleLookup(const Slice& key, Statistics* stats) { + Handle* h = key_only_cache_->Lookup(key); + if (h != nullptr) { + key_only_cache_->Release(h); + inc_hit_counter(); + RecordTick(stats, SIM_BLOCK_CACHE_HIT); + } else { + inc_miss_counter(); + RecordTick(stats, SIM_BLOCK_CACHE_MISS); + } + cache_activity_logger_.ReportLookup(key); + } }; } // end anonymous namespace