2016-08-19 19:28:19 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2016-08-19 19:28:19 +00:00
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2017-04-06 02:02:00 +00:00
|
|
|
#include "cache/clock_cache.h"
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
#include <cassert>
|
|
|
|
#include <cstdint>
|
|
|
|
#include <cstdio>
|
|
|
|
#include <functional>
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
#include "monitoring/perf_context_imp.h"
|
|
|
|
#include "monitoring/statistics.h"
|
|
|
|
#include "port/lang.h"
|
|
|
|
#include "util/hash.h"
|
|
|
|
#include "util/math.h"
|
|
|
|
#include "util/random.h"
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
namespace clock_cache {
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
ClockHandleTable::ClockHandleTable(size_t capacity, int hash_bits)
|
2022-06-30 04:50:39 +00:00
|
|
|
: length_bits_(hash_bits),
|
|
|
|
length_bits_mask_((uint32_t{1} << length_bits_) - 1),
|
|
|
|
occupancy_limit_(static_cast<uint32_t>((uint32_t{1} << length_bits_) *
|
|
|
|
kStrictLoadFactor)),
|
2022-07-25 17:02:19 +00:00
|
|
|
capacity_(capacity),
|
|
|
|
array_(new ClockHandle[size_t{1} << length_bits_]),
|
|
|
|
clock_pointer_(0),
|
|
|
|
occupancy_(0),
|
|
|
|
usage_(0) {
|
2022-06-30 04:50:39 +00:00
|
|
|
assert(hash_bits <= 32);
|
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
ClockHandleTable::~ClockHandleTable() {
|
2022-07-25 17:02:19 +00:00
|
|
|
// Assumes there are no references (of any type) to any slot in the table.
|
|
|
|
for (uint32_t i = 0; i < GetTableSize(); i++) {
|
|
|
|
ClockHandle* h = &array_[i];
|
|
|
|
if (h->IsElement()) {
|
|
|
|
h->FreeData();
|
|
|
|
}
|
|
|
|
}
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-16 05:36:58 +00:00
|
|
|
ClockHandle* ClockHandleTable::Lookup(const Slice& key, uint32_t hash) {
|
|
|
|
uint32_t probe = 0;
|
2022-07-25 17:02:19 +00:00
|
|
|
ClockHandle* e = FindSlot(
|
|
|
|
key,
|
|
|
|
[&](ClockHandle* h) {
|
|
|
|
if (h->TryInternalRef()) {
|
|
|
|
if (h->IsElement() && h->Matches(key, hash)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
h->ReleaseInternalRef();
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
},
|
|
|
|
[&](ClockHandle* h) { return h->displacements == 0; },
|
|
|
|
[&](ClockHandle* /*h*/) {}, probe);
|
|
|
|
|
|
|
|
if (e != nullptr) {
|
|
|
|
// TODO(Guido) Comment from #10347: Here it looks like we have three atomic
|
|
|
|
// updates where it would be possible to combine into one CAS (more metadata
|
|
|
|
// under one atomic field) or maybe two atomic updates (one arithmetic, one
|
|
|
|
// bitwise). Something to think about optimizing.
|
|
|
|
e->SetHit();
|
|
|
|
// The handle is now referenced, so we take it out of clock.
|
|
|
|
ClockOff(e);
|
2022-07-27 00:42:03 +00:00
|
|
|
e->InternalToExternalRef();
|
2022-07-25 17:02:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return e;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
ClockHandle* ClockHandleTable::Insert(ClockHandle* h,
|
|
|
|
autovector<ClockHandle>* deleted,
|
|
|
|
bool take_reference) {
|
2022-07-16 05:36:58 +00:00
|
|
|
uint32_t probe = 0;
|
2022-07-25 17:02:19 +00:00
|
|
|
ClockHandle* e = FindAvailableSlot(h->key(), h->hash, probe, deleted);
|
|
|
|
if (e == nullptr) {
|
|
|
|
// No available slot to place the handle.
|
2022-06-30 04:50:39 +00:00
|
|
|
return nullptr;
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
2019-09-16 22:14:51 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
// The slot is empty or is a tombstone. And we have an exclusive ref.
|
|
|
|
Assign(e, h);
|
|
|
|
// TODO(Guido) The following RemoveAll can probably be run outside of
|
|
|
|
// the exclusive ref. I had a bad case in mind: multiple inserts could
|
|
|
|
// annihilate each. Although I think this is impossible, I'm not sure
|
|
|
|
// my mental proof covers every case.
|
|
|
|
if (e->displacements != 0) {
|
|
|
|
// It used to be a tombstone, so there may already be copies of the
|
2022-06-30 04:50:39 +00:00
|
|
|
// key in the table.
|
2022-07-25 17:02:19 +00:00
|
|
|
RemoveAll(h->key(), h->hash, probe, deleted);
|
2019-09-16 22:14:51 +00:00
|
|
|
}
|
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
if (take_reference) {
|
|
|
|
// The user wants to take a reference.
|
|
|
|
e->ExclusiveToExternalRef();
|
|
|
|
} else {
|
|
|
|
// The user doesn't want to immediately take a reference, so we make
|
|
|
|
// it evictable.
|
|
|
|
ClockOn(e);
|
|
|
|
e->ReleaseExclusiveRef();
|
|
|
|
}
|
|
|
|
return e;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-16 05:36:58 +00:00
|
|
|
void ClockHandleTable::Assign(ClockHandle* dst, ClockHandle* src) {
|
|
|
|
// DON'T touch displacements and refs.
|
|
|
|
dst->value = src->value;
|
|
|
|
dst->deleter = src->deleter;
|
|
|
|
dst->hash = src->hash;
|
|
|
|
dst->total_charge = src->total_charge;
|
|
|
|
dst->key_data = src->key_data;
|
|
|
|
dst->flags.store(0);
|
2022-06-30 04:50:39 +00:00
|
|
|
dst->SetIsElement(true);
|
2022-07-16 05:36:58 +00:00
|
|
|
dst->SetCachePriority(src->GetCachePriority());
|
2022-07-25 17:02:19 +00:00
|
|
|
usage_ += dst->total_charge;
|
2022-06-30 04:50:39 +00:00
|
|
|
occupancy_++;
|
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
bool ClockHandleTable::TryRemove(ClockHandle* h,
|
|
|
|
autovector<ClockHandle>* deleted) {
|
|
|
|
if (h->TryExclusiveRef()) {
|
|
|
|
if (h->WillBeDeleted()) {
|
|
|
|
Remove(h, deleted);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
h->ReleaseExclusiveRef();
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ClockHandleTable::SpinTryRemove(ClockHandle* h,
|
|
|
|
autovector<ClockHandle>* deleted) {
|
|
|
|
if (h->SpinTryExclusiveRef()) {
|
|
|
|
if (h->WillBeDeleted()) {
|
|
|
|
Remove(h, deleted);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
h->ReleaseExclusiveRef();
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ClockHandleTable::ClockOff(ClockHandle* h) {
|
|
|
|
h->SetClockPriority(ClockHandle::ClockPriority::NONE);
|
|
|
|
}
|
|
|
|
|
|
|
|
void ClockHandleTable::ClockOn(ClockHandle* h) {
|
|
|
|
assert(!h->IsInClock());
|
|
|
|
bool is_high_priority =
|
|
|
|
h->HasHit() || h->GetCachePriority() == Cache::Priority::HIGH;
|
|
|
|
h->SetClockPriority(static_cast<ClockHandle::ClockPriority>(
|
|
|
|
is_high_priority ? ClockHandle::ClockPriority::HIGH
|
|
|
|
: ClockHandle::ClockPriority::MEDIUM));
|
|
|
|
}
|
|
|
|
|
|
|
|
void ClockHandleTable::Remove(ClockHandle* h,
|
|
|
|
autovector<ClockHandle>* deleted) {
|
|
|
|
deleted->push_back(*h);
|
|
|
|
ClockOff(h);
|
|
|
|
uint32_t probe = 0;
|
|
|
|
FindSlot(
|
|
|
|
h->key(), [&](ClockHandle* e) { return e == h; },
|
|
|
|
[&](ClockHandle* /*e*/) { return false; },
|
|
|
|
[&](ClockHandle* e) { e->displacements--; }, probe);
|
|
|
|
h->SetWillBeDeleted(false);
|
|
|
|
h->SetIsElement(false);
|
|
|
|
}
|
|
|
|
|
|
|
|
void ClockHandleTable::RemoveAll(const Slice& key, uint32_t hash,
|
|
|
|
uint32_t& probe,
|
|
|
|
autovector<ClockHandle>* deleted) {
|
|
|
|
FindSlot(
|
2022-07-16 05:36:58 +00:00
|
|
|
key,
|
|
|
|
[&](ClockHandle* h) {
|
|
|
|
if (h->TryInternalRef()) {
|
2022-07-25 17:02:19 +00:00
|
|
|
if (h->IsElement() && h->Matches(key, hash)) {
|
|
|
|
h->SetWillBeDeleted(true);
|
|
|
|
h->ReleaseInternalRef();
|
|
|
|
if (TryRemove(h, deleted)) {
|
|
|
|
h->ReleaseExclusiveRef();
|
|
|
|
}
|
|
|
|
return false;
|
2022-07-16 05:36:58 +00:00
|
|
|
}
|
|
|
|
h->ReleaseInternalRef();
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
},
|
|
|
|
[&](ClockHandle* h) { return h->displacements == 0; },
|
|
|
|
[&](ClockHandle* /*h*/) {}, probe);
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
void ClockHandleTable::Free(autovector<ClockHandle>* deleted) {
|
|
|
|
if (deleted->size() == 0) {
|
|
|
|
// Avoid unnecessarily reading usage_ and occupancy_.
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t deleted_charge = 0;
|
|
|
|
for (auto& h : *deleted) {
|
|
|
|
deleted_charge += h.total_charge;
|
|
|
|
h.FreeData();
|
2022-07-16 05:36:58 +00:00
|
|
|
}
|
2022-07-25 17:02:19 +00:00
|
|
|
assert(usage_ >= deleted_charge);
|
|
|
|
usage_ -= deleted_charge;
|
|
|
|
occupancy_ -= static_cast<uint32_t>(deleted->size());
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
ClockHandle* ClockHandleTable::FindAvailableSlot(
|
|
|
|
const Slice& key, uint32_t hash, uint32_t& probe,
|
|
|
|
autovector<ClockHandle>* deleted) {
|
|
|
|
ClockHandle* e = FindSlot(
|
2022-06-30 04:50:39 +00:00
|
|
|
key,
|
|
|
|
[&](ClockHandle* h) {
|
2022-07-25 17:02:19 +00:00
|
|
|
// To read the handle, first acquire a shared ref.
|
2022-07-16 05:36:58 +00:00
|
|
|
if (h->TryInternalRef()) {
|
2022-07-25 17:02:19 +00:00
|
|
|
if (h->IsElement()) {
|
|
|
|
// The slot is not available.
|
|
|
|
// TODO(Guido) Is it worth testing h->WillBeDeleted()?
|
|
|
|
if (h->WillBeDeleted() || h->Matches(key, hash)) {
|
|
|
|
// The slot can be freed up, or the key we're inserting is already
|
|
|
|
// in the table, so we try to delete it. When the attempt is
|
|
|
|
// successful, the slot becomes available, so we stop probing.
|
|
|
|
// Notice that in that case TryRemove returns an exclusive ref.
|
|
|
|
h->SetWillBeDeleted(true);
|
|
|
|
h->ReleaseInternalRef();
|
|
|
|
if (TryRemove(h, deleted)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
h->ReleaseInternalRef();
|
|
|
|
return false;
|
2022-07-16 05:36:58 +00:00
|
|
|
}
|
2022-07-25 17:02:19 +00:00
|
|
|
|
|
|
|
// Available slot.
|
2022-07-16 05:36:58 +00:00
|
|
|
h->ReleaseInternalRef();
|
2022-07-25 17:02:19 +00:00
|
|
|
// Try to acquire an exclusive ref. If we fail, continue probing.
|
|
|
|
if (h->SpinTryExclusiveRef()) {
|
|
|
|
// Check that the slot is still available.
|
|
|
|
if (!h->IsElement()) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
h->ReleaseExclusiveRef();
|
|
|
|
}
|
2022-07-16 05:36:58 +00:00
|
|
|
}
|
|
|
|
return false;
|
2022-06-30 04:50:39 +00:00
|
|
|
},
|
2022-07-16 05:36:58 +00:00
|
|
|
[&](ClockHandle* /*h*/) { return false; },
|
|
|
|
[&](ClockHandle* h) { h->displacements++; }, probe);
|
2022-07-25 17:02:19 +00:00
|
|
|
if (e == nullptr) {
|
2022-07-16 05:36:58 +00:00
|
|
|
Rollback(key, probe);
|
|
|
|
}
|
2022-07-25 17:02:19 +00:00
|
|
|
return e;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
ClockHandle* ClockHandleTable::FindSlot(
|
|
|
|
const Slice& key, std::function<bool(ClockHandle*)> match,
|
|
|
|
std::function<bool(ClockHandle*)> abort,
|
|
|
|
std::function<void(ClockHandle*)> update, uint32_t& probe) {
|
2022-07-16 05:36:58 +00:00
|
|
|
// We use double-hashing probing. Every probe in the sequence is a
|
|
|
|
// pseudorandom integer, computed as a linear function of two random hashes,
|
|
|
|
// which we call base and increment. Specifically, the i-th probe is base + i
|
|
|
|
// * increment modulo the table size.
|
2022-06-30 04:50:39 +00:00
|
|
|
uint32_t base = ModTableSize(Hash(key.data(), key.size(), kProbingSeed1));
|
2022-07-16 05:36:58 +00:00
|
|
|
// We use an odd increment, which is relatively prime with the power-of-two
|
|
|
|
// table size. This implies that we cycle back to the first probe only
|
|
|
|
// after probing every slot exactly once.
|
2022-06-30 04:50:39 +00:00
|
|
|
uint32_t increment =
|
|
|
|
ModTableSize((Hash(key.data(), key.size(), kProbingSeed2) << 1) | 1);
|
|
|
|
uint32_t current = ModTableSize(base + probe * increment);
|
|
|
|
while (true) {
|
|
|
|
ClockHandle* h = &array_[current];
|
2022-07-16 05:36:58 +00:00
|
|
|
if (current == base && probe > 0) {
|
2022-06-30 04:50:39 +00:00
|
|
|
// We looped back.
|
2022-07-25 17:02:19 +00:00
|
|
|
return nullptr;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2022-07-16 05:36:58 +00:00
|
|
|
if (match(h)) {
|
|
|
|
probe++;
|
2022-07-25 17:02:19 +00:00
|
|
|
return h;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2022-07-16 05:36:58 +00:00
|
|
|
if (abort(h)) {
|
2022-07-25 17:02:19 +00:00
|
|
|
return nullptr;
|
2016-08-31 15:56:34 +00:00
|
|
|
}
|
2022-07-16 05:36:58 +00:00
|
|
|
probe++;
|
|
|
|
update(h);
|
|
|
|
current = ModTableSize(current + increment);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void ClockHandleTable::Rollback(const Slice& key, uint32_t probe) {
|
|
|
|
uint32_t current = ModTableSize(Hash(key.data(), key.size(), kProbingSeed1));
|
|
|
|
uint32_t increment =
|
|
|
|
ModTableSize((Hash(key.data(), key.size(), kProbingSeed2) << 1) | 1);
|
|
|
|
for (uint32_t i = 0; i < probe; i++) {
|
|
|
|
array_[current].displacements--;
|
2022-06-30 04:50:39 +00:00
|
|
|
current = ModTableSize(current + increment);
|
2016-08-31 15:56:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
void ClockHandleTable::ClockRun(size_t charge) {
|
|
|
|
// TODO(Guido) When an element is in the probe sequence of a
|
|
|
|
// hot element, it will be hard to get an exclusive ref.
|
|
|
|
// Do we need a mechanism to prevent an element from sitting
|
|
|
|
// for a long time in cache waiting to be evicted?
|
|
|
|
autovector<ClockHandle> deleted;
|
|
|
|
uint32_t max_iterations =
|
2022-07-27 00:42:03 +00:00
|
|
|
ClockHandle::ClockPriority::HIGH *
|
|
|
|
(1 +
|
|
|
|
static_cast<uint32_t>(
|
|
|
|
GetTableSize() *
|
|
|
|
kLoadFactor)); // It may take up to HIGH passes to evict an element.
|
2022-07-25 17:02:19 +00:00
|
|
|
size_t usage_local = usage_;
|
2022-07-27 00:42:03 +00:00
|
|
|
size_t capacity_local = capacity_;
|
|
|
|
while (usage_local + charge > capacity_local && max_iterations--) {
|
2022-07-25 17:02:19 +00:00
|
|
|
uint32_t steps = 1 + static_cast<uint32_t>(1 / kLoadFactor);
|
|
|
|
uint32_t clock_pointer_local = (clock_pointer_ += steps) - steps;
|
|
|
|
for (uint32_t i = 0; i < steps; i++) {
|
|
|
|
ClockHandle* h = &array_[ModTableSize(clock_pointer_local + i)];
|
|
|
|
if (h->TryExclusiveRef()) {
|
|
|
|
if (h->WillBeDeleted()) {
|
|
|
|
Remove(h, &deleted);
|
|
|
|
usage_local -= h->total_charge;
|
|
|
|
} else {
|
|
|
|
if (!h->IsInClock() && h->IsElement()) {
|
|
|
|
// We adjust the clock priority to make the element evictable again.
|
|
|
|
// Why? Elements that are not in clock are either currently
|
|
|
|
// externally referenced or used to be. Because we are holding an
|
|
|
|
// exclusive ref, we know we are in the latter case. This can only
|
|
|
|
// happen when the last external reference to an element was
|
|
|
|
// released, and the element was not immediately removed.
|
|
|
|
ClockOn(h);
|
|
|
|
}
|
|
|
|
ClockHandle::ClockPriority priority = h->GetClockPriority();
|
|
|
|
if (priority == ClockHandle::ClockPriority::LOW) {
|
|
|
|
Remove(h, &deleted);
|
|
|
|
usage_local -= h->total_charge;
|
|
|
|
} else if (priority > ClockHandle::ClockPriority::LOW) {
|
|
|
|
h->DecreaseClockPriority();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
h->ReleaseExclusiveRef();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Free(&deleted);
|
|
|
|
}
|
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
ClockCacheShard::ClockCacheShard(
|
|
|
|
size_t capacity, size_t estimated_value_size, bool strict_capacity_limit,
|
|
|
|
CacheMetadataChargePolicy metadata_charge_policy)
|
2022-07-25 17:02:19 +00:00
|
|
|
: strict_capacity_limit_(strict_capacity_limit),
|
2022-07-27 00:42:03 +00:00
|
|
|
detached_usage_(0),
|
2022-07-25 17:02:19 +00:00
|
|
|
table_(capacity, CalcHashBits(capacity, estimated_value_size,
|
|
|
|
metadata_charge_policy)) {
|
2022-06-30 04:50:39 +00:00
|
|
|
set_metadata_charge_policy(metadata_charge_policy);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
void ClockCacheShard::EraseUnRefEntries() {
|
2022-07-25 17:02:19 +00:00
|
|
|
autovector<ClockHandle> deleted;
|
2022-06-30 04:50:39 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
table_.ApplyToEntriesRange(
|
|
|
|
[this, &deleted](ClockHandle* h) {
|
|
|
|
// Externally unreferenced element.
|
|
|
|
table_.Remove(h, &deleted);
|
|
|
|
},
|
|
|
|
0, table_.GetTableSize(), true);
|
|
|
|
|
|
|
|
table_.Free(&deleted);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
New Cache API for gathering statistics (#8225)
Summary:
Adds a new Cache::ApplyToAllEntries API that we expect to use
(in follow-up PRs) for efficiently gathering block cache statistics.
Notable features vs. old ApplyToAllCacheEntries:
* Includes key and deleter (in addition to value and charge). We could
have passed in a Handle but then more virtual function calls would be
needed to get the "fields" of each entry. We expect to use the 'deleter'
to identify the origin of entries, perhaps even more.
* Heavily tuned to minimize latency impact on operating cache. It
does this by iterating over small sections of each cache shard while
cycling through the shards.
* Supports tuning roughly how many entries to operate on for each
lock acquire and release, to control the impact on the latency of other
operations without excessive lock acquire & release. The right balance
can depend on the cost of the callback. Good default seems to be
around 256.
* There should be no need to disable thread safety. (I would expect
uncontended locks to be sufficiently fast.)
I have enhanced cache_bench to validate this approach:
* Reports a histogram of ns per operation, so we can look at the
ditribution of times, not just throughput (average).
* Can add a thread for simulated "gather stats" which calls
ApplyToAllEntries at a specified interval. We also generate a histogram
of time to run ApplyToAllEntries.
To make the iteration over some entries of each shard work as cleanly as
possible, even with resize between next set of entries, I have
re-arranged which hash bits are used for sharding and which for indexing
within a shard.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8225
Test Plan:
A couple of unit tests are added, but primary validation is manual, as
the primary risk is to performance.
The primary validation is using cache_bench to ensure that neither
the minor hashing changes nor the simulated stats gathering
significantly impact QPS or latency distribution. Note that adding op
latency histogram seriously impacts the benchmark QPS, so for a
fair baseline, we need the cache_bench changes (except remove simulated
stat gathering to make it compile). In short, we don't see any
reproducible difference in ops/sec or op latency unless we are gathering
stats nearly continuously. Test uses 10GB block cache with
8KB values to be somewhat realistic in the number of items to iterate
over.
Baseline typical output:
```
Complete in 92.017 s; Rough parallel ops/sec = 869401
Thread ops/sec = 54662
Operation latency (ns):
Count: 80000000 Average: 11223.9494 StdDev: 29.61
Min: 0 Median: 7759.3973 Max: 9620500
Percentiles: P50: 7759.40 P75: 14190.73 P99: 46922.75 P99.9: 77509.84 P99.99: 217030.58
------------------------------------------------------
[ 0, 1 ] 68 0.000% 0.000%
( 2900, 4400 ] 89 0.000% 0.000%
( 4400, 6600 ] 33630240 42.038% 42.038% ########
( 6600, 9900 ] 18129842 22.662% 64.700% #####
( 9900, 14000 ] 7877533 9.847% 74.547% ##
( 14000, 22000 ] 15193238 18.992% 93.539% ####
( 22000, 33000 ] 3037061 3.796% 97.335% #
( 33000, 50000 ] 1626316 2.033% 99.368%
( 50000, 75000 ] 421532 0.527% 99.895%
( 75000, 110000 ] 56910 0.071% 99.966%
( 110000, 170000 ] 16134 0.020% 99.986%
( 170000, 250000 ] 5166 0.006% 99.993%
( 250000, 380000 ] 3017 0.004% 99.996%
( 380000, 570000 ] 1337 0.002% 99.998%
( 570000, 860000 ] 805 0.001% 99.999%
( 860000, 1200000 ] 319 0.000% 100.000%
( 1200000, 1900000 ] 231 0.000% 100.000%
( 1900000, 2900000 ] 100 0.000% 100.000%
( 2900000, 4300000 ] 39 0.000% 100.000%
( 4300000, 6500000 ] 16 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
```
New, gather_stats=false. Median thread ops/sec of 5 runs:
```
Complete in 92.030 s; Rough parallel ops/sec = 869285
Thread ops/sec = 54458
Operation latency (ns):
Count: 80000000 Average: 11298.1027 StdDev: 42.18
Min: 0 Median: 7722.0822 Max: 6398720
Percentiles: P50: 7722.08 P75: 14294.68 P99: 47522.95 P99.9: 85292.16 P99.99: 228077.78
------------------------------------------------------
[ 0, 1 ] 109 0.000% 0.000%
( 2900, 4400 ] 793 0.001% 0.001%
( 4400, 6600 ] 34054563 42.568% 42.569% #########
( 6600, 9900 ] 17482646 21.853% 64.423% ####
( 9900, 14000 ] 7908180 9.885% 74.308% ##
( 14000, 22000 ] 15032072 18.790% 93.098% ####
( 22000, 33000 ] 3237834 4.047% 97.145% #
( 33000, 50000 ] 1736882 2.171% 99.316%
( 50000, 75000 ] 446851 0.559% 99.875%
( 75000, 110000 ] 68251 0.085% 99.960%
( 110000, 170000 ] 18592 0.023% 99.983%
( 170000, 250000 ] 7200 0.009% 99.992%
( 250000, 380000 ] 3334 0.004% 99.997%
( 380000, 570000 ] 1393 0.002% 99.998%
( 570000, 860000 ] 700 0.001% 99.999%
( 860000, 1200000 ] 293 0.000% 100.000%
( 1200000, 1900000 ] 196 0.000% 100.000%
( 1900000, 2900000 ] 69 0.000% 100.000%
( 2900000, 4300000 ] 32 0.000% 100.000%
( 4300000, 6500000 ] 10 0.000% 100.000%
```
New, gather_stats=true, 1 second delay between scans. Scans take about
1 second here so it's spending about 50% time scanning. Still the effect on
ops/sec and latency seems to be in the noise. Median thread ops/sec of 5 runs:
```
Complete in 91.890 s; Rough parallel ops/sec = 870608
Thread ops/sec = 54551
Operation latency (ns):
Count: 80000000 Average: 11311.2629 StdDev: 45.28
Min: 0 Median: 7686.5458 Max: 10018340
Percentiles: P50: 7686.55 P75: 14481.95 P99: 47232.60 P99.9: 79230.18 P99.99: 232998.86
------------------------------------------------------
[ 0, 1 ] 71 0.000% 0.000%
( 2900, 4400 ] 291 0.000% 0.000%
( 4400, 6600 ] 34492060 43.115% 43.116% #########
( 6600, 9900 ] 16727328 20.909% 64.025% ####
( 9900, 14000 ] 7845828 9.807% 73.832% ##
( 14000, 22000 ] 15510654 19.388% 93.220% ####
( 22000, 33000 ] 3216533 4.021% 97.241% #
( 33000, 50000 ] 1680859 2.101% 99.342%
( 50000, 75000 ] 439059 0.549% 99.891%
( 75000, 110000 ] 60540 0.076% 99.967%
( 110000, 170000 ] 14649 0.018% 99.985%
( 170000, 250000 ] 5242 0.007% 99.991%
( 250000, 380000 ] 3260 0.004% 99.995%
( 380000, 570000 ] 1599 0.002% 99.997%
( 570000, 860000 ] 1043 0.001% 99.999%
( 860000, 1200000 ] 471 0.001% 99.999%
( 1200000, 1900000 ] 275 0.000% 100.000%
( 1900000, 2900000 ] 143 0.000% 100.000%
( 2900000, 4300000 ] 60 0.000% 100.000%
( 4300000, 6500000 ] 27 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
( 9800000, 14000000 ] 1 0.000% 100.000%
Gather stats latency (us):
Count: 46 Average: 980387.5870 StdDev: 60911.18
Min: 879155 Median: 1033777.7778 Max: 1261431
Percentiles: P50: 1033777.78 P75: 1120666.67 P99: 1261431.00 P99.9: 1261431.00 P99.99: 1261431.00
------------------------------------------------------
( 860000, 1200000 ] 45 97.826% 97.826% ####################
( 1200000, 1900000 ] 1 2.174% 100.000%
Most recent cache entry stats:
Number of entries: 1295133
Total charge: 9.88 GB
Average key size: 23.4982
Average charge: 8.00 KB
Unique deleters: 3
```
Reviewed By: mrambacher
Differential Revision: D28295742
Pulled By: pdillinger
fbshipit-source-id: bbc4a552f91ba0fe10e5cc025c42cef5a81f2b95
2021-05-11 23:16:11 +00:00
|
|
|
void ClockCacheShard::ApplyToSomeEntries(
|
|
|
|
const std::function<void(const Slice& key, void* value, size_t charge,
|
|
|
|
DeleterFn deleter)>& callback,
|
|
|
|
uint32_t average_entries_per_lock, uint32_t* state) {
|
2022-06-30 04:50:39 +00:00
|
|
|
// The state is essentially going to be the starting hash, which works
|
|
|
|
// nicely even if we resize between calls because we use upper-most
|
|
|
|
// hash bits for table indexes.
|
|
|
|
uint32_t length_bits = table_.GetLengthBits();
|
|
|
|
uint32_t length = table_.GetTableSize();
|
New Cache API for gathering statistics (#8225)
Summary:
Adds a new Cache::ApplyToAllEntries API that we expect to use
(in follow-up PRs) for efficiently gathering block cache statistics.
Notable features vs. old ApplyToAllCacheEntries:
* Includes key and deleter (in addition to value and charge). We could
have passed in a Handle but then more virtual function calls would be
needed to get the "fields" of each entry. We expect to use the 'deleter'
to identify the origin of entries, perhaps even more.
* Heavily tuned to minimize latency impact on operating cache. It
does this by iterating over small sections of each cache shard while
cycling through the shards.
* Supports tuning roughly how many entries to operate on for each
lock acquire and release, to control the impact on the latency of other
operations without excessive lock acquire & release. The right balance
can depend on the cost of the callback. Good default seems to be
around 256.
* There should be no need to disable thread safety. (I would expect
uncontended locks to be sufficiently fast.)
I have enhanced cache_bench to validate this approach:
* Reports a histogram of ns per operation, so we can look at the
ditribution of times, not just throughput (average).
* Can add a thread for simulated "gather stats" which calls
ApplyToAllEntries at a specified interval. We also generate a histogram
of time to run ApplyToAllEntries.
To make the iteration over some entries of each shard work as cleanly as
possible, even with resize between next set of entries, I have
re-arranged which hash bits are used for sharding and which for indexing
within a shard.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8225
Test Plan:
A couple of unit tests are added, but primary validation is manual, as
the primary risk is to performance.
The primary validation is using cache_bench to ensure that neither
the minor hashing changes nor the simulated stats gathering
significantly impact QPS or latency distribution. Note that adding op
latency histogram seriously impacts the benchmark QPS, so for a
fair baseline, we need the cache_bench changes (except remove simulated
stat gathering to make it compile). In short, we don't see any
reproducible difference in ops/sec or op latency unless we are gathering
stats nearly continuously. Test uses 10GB block cache with
8KB values to be somewhat realistic in the number of items to iterate
over.
Baseline typical output:
```
Complete in 92.017 s; Rough parallel ops/sec = 869401
Thread ops/sec = 54662
Operation latency (ns):
Count: 80000000 Average: 11223.9494 StdDev: 29.61
Min: 0 Median: 7759.3973 Max: 9620500
Percentiles: P50: 7759.40 P75: 14190.73 P99: 46922.75 P99.9: 77509.84 P99.99: 217030.58
------------------------------------------------------
[ 0, 1 ] 68 0.000% 0.000%
( 2900, 4400 ] 89 0.000% 0.000%
( 4400, 6600 ] 33630240 42.038% 42.038% ########
( 6600, 9900 ] 18129842 22.662% 64.700% #####
( 9900, 14000 ] 7877533 9.847% 74.547% ##
( 14000, 22000 ] 15193238 18.992% 93.539% ####
( 22000, 33000 ] 3037061 3.796% 97.335% #
( 33000, 50000 ] 1626316 2.033% 99.368%
( 50000, 75000 ] 421532 0.527% 99.895%
( 75000, 110000 ] 56910 0.071% 99.966%
( 110000, 170000 ] 16134 0.020% 99.986%
( 170000, 250000 ] 5166 0.006% 99.993%
( 250000, 380000 ] 3017 0.004% 99.996%
( 380000, 570000 ] 1337 0.002% 99.998%
( 570000, 860000 ] 805 0.001% 99.999%
( 860000, 1200000 ] 319 0.000% 100.000%
( 1200000, 1900000 ] 231 0.000% 100.000%
( 1900000, 2900000 ] 100 0.000% 100.000%
( 2900000, 4300000 ] 39 0.000% 100.000%
( 4300000, 6500000 ] 16 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
```
New, gather_stats=false. Median thread ops/sec of 5 runs:
```
Complete in 92.030 s; Rough parallel ops/sec = 869285
Thread ops/sec = 54458
Operation latency (ns):
Count: 80000000 Average: 11298.1027 StdDev: 42.18
Min: 0 Median: 7722.0822 Max: 6398720
Percentiles: P50: 7722.08 P75: 14294.68 P99: 47522.95 P99.9: 85292.16 P99.99: 228077.78
------------------------------------------------------
[ 0, 1 ] 109 0.000% 0.000%
( 2900, 4400 ] 793 0.001% 0.001%
( 4400, 6600 ] 34054563 42.568% 42.569% #########
( 6600, 9900 ] 17482646 21.853% 64.423% ####
( 9900, 14000 ] 7908180 9.885% 74.308% ##
( 14000, 22000 ] 15032072 18.790% 93.098% ####
( 22000, 33000 ] 3237834 4.047% 97.145% #
( 33000, 50000 ] 1736882 2.171% 99.316%
( 50000, 75000 ] 446851 0.559% 99.875%
( 75000, 110000 ] 68251 0.085% 99.960%
( 110000, 170000 ] 18592 0.023% 99.983%
( 170000, 250000 ] 7200 0.009% 99.992%
( 250000, 380000 ] 3334 0.004% 99.997%
( 380000, 570000 ] 1393 0.002% 99.998%
( 570000, 860000 ] 700 0.001% 99.999%
( 860000, 1200000 ] 293 0.000% 100.000%
( 1200000, 1900000 ] 196 0.000% 100.000%
( 1900000, 2900000 ] 69 0.000% 100.000%
( 2900000, 4300000 ] 32 0.000% 100.000%
( 4300000, 6500000 ] 10 0.000% 100.000%
```
New, gather_stats=true, 1 second delay between scans. Scans take about
1 second here so it's spending about 50% time scanning. Still the effect on
ops/sec and latency seems to be in the noise. Median thread ops/sec of 5 runs:
```
Complete in 91.890 s; Rough parallel ops/sec = 870608
Thread ops/sec = 54551
Operation latency (ns):
Count: 80000000 Average: 11311.2629 StdDev: 45.28
Min: 0 Median: 7686.5458 Max: 10018340
Percentiles: P50: 7686.55 P75: 14481.95 P99: 47232.60 P99.9: 79230.18 P99.99: 232998.86
------------------------------------------------------
[ 0, 1 ] 71 0.000% 0.000%
( 2900, 4400 ] 291 0.000% 0.000%
( 4400, 6600 ] 34492060 43.115% 43.116% #########
( 6600, 9900 ] 16727328 20.909% 64.025% ####
( 9900, 14000 ] 7845828 9.807% 73.832% ##
( 14000, 22000 ] 15510654 19.388% 93.220% ####
( 22000, 33000 ] 3216533 4.021% 97.241% #
( 33000, 50000 ] 1680859 2.101% 99.342%
( 50000, 75000 ] 439059 0.549% 99.891%
( 75000, 110000 ] 60540 0.076% 99.967%
( 110000, 170000 ] 14649 0.018% 99.985%
( 170000, 250000 ] 5242 0.007% 99.991%
( 250000, 380000 ] 3260 0.004% 99.995%
( 380000, 570000 ] 1599 0.002% 99.997%
( 570000, 860000 ] 1043 0.001% 99.999%
( 860000, 1200000 ] 471 0.001% 99.999%
( 1200000, 1900000 ] 275 0.000% 100.000%
( 1900000, 2900000 ] 143 0.000% 100.000%
( 2900000, 4300000 ] 60 0.000% 100.000%
( 4300000, 6500000 ] 27 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
( 9800000, 14000000 ] 1 0.000% 100.000%
Gather stats latency (us):
Count: 46 Average: 980387.5870 StdDev: 60911.18
Min: 879155 Median: 1033777.7778 Max: 1261431
Percentiles: P50: 1033777.78 P75: 1120666.67 P99: 1261431.00 P99.9: 1261431.00 P99.99: 1261431.00
------------------------------------------------------
( 860000, 1200000 ] 45 97.826% 97.826% ####################
( 1200000, 1900000 ] 1 2.174% 100.000%
Most recent cache entry stats:
Number of entries: 1295133
Total charge: 9.88 GB
Average key size: 23.4982
Average charge: 8.00 KB
Unique deleters: 3
```
Reviewed By: mrambacher
Differential Revision: D28295742
Pulled By: pdillinger
fbshipit-source-id: bbc4a552f91ba0fe10e5cc025c42cef5a81f2b95
2021-05-11 23:16:11 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
assert(average_entries_per_lock > 0);
|
|
|
|
// Assuming we are called with same average_entries_per_lock repeatedly,
|
|
|
|
// this simplifies some logic (index_end will not overflow).
|
|
|
|
assert(average_entries_per_lock < length || *state == 0);
|
|
|
|
|
|
|
|
uint32_t index_begin = *state >> (32 - length_bits);
|
|
|
|
uint32_t index_end = index_begin + average_entries_per_lock;
|
|
|
|
if (index_end >= length) {
|
2022-07-25 17:02:19 +00:00
|
|
|
// Going to end.
|
2022-06-30 04:50:39 +00:00
|
|
|
index_end = length;
|
New Cache API for gathering statistics (#8225)
Summary:
Adds a new Cache::ApplyToAllEntries API that we expect to use
(in follow-up PRs) for efficiently gathering block cache statistics.
Notable features vs. old ApplyToAllCacheEntries:
* Includes key and deleter (in addition to value and charge). We could
have passed in a Handle but then more virtual function calls would be
needed to get the "fields" of each entry. We expect to use the 'deleter'
to identify the origin of entries, perhaps even more.
* Heavily tuned to minimize latency impact on operating cache. It
does this by iterating over small sections of each cache shard while
cycling through the shards.
* Supports tuning roughly how many entries to operate on for each
lock acquire and release, to control the impact on the latency of other
operations without excessive lock acquire & release. The right balance
can depend on the cost of the callback. Good default seems to be
around 256.
* There should be no need to disable thread safety. (I would expect
uncontended locks to be sufficiently fast.)
I have enhanced cache_bench to validate this approach:
* Reports a histogram of ns per operation, so we can look at the
ditribution of times, not just throughput (average).
* Can add a thread for simulated "gather stats" which calls
ApplyToAllEntries at a specified interval. We also generate a histogram
of time to run ApplyToAllEntries.
To make the iteration over some entries of each shard work as cleanly as
possible, even with resize between next set of entries, I have
re-arranged which hash bits are used for sharding and which for indexing
within a shard.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8225
Test Plan:
A couple of unit tests are added, but primary validation is manual, as
the primary risk is to performance.
The primary validation is using cache_bench to ensure that neither
the minor hashing changes nor the simulated stats gathering
significantly impact QPS or latency distribution. Note that adding op
latency histogram seriously impacts the benchmark QPS, so for a
fair baseline, we need the cache_bench changes (except remove simulated
stat gathering to make it compile). In short, we don't see any
reproducible difference in ops/sec or op latency unless we are gathering
stats nearly continuously. Test uses 10GB block cache with
8KB values to be somewhat realistic in the number of items to iterate
over.
Baseline typical output:
```
Complete in 92.017 s; Rough parallel ops/sec = 869401
Thread ops/sec = 54662
Operation latency (ns):
Count: 80000000 Average: 11223.9494 StdDev: 29.61
Min: 0 Median: 7759.3973 Max: 9620500
Percentiles: P50: 7759.40 P75: 14190.73 P99: 46922.75 P99.9: 77509.84 P99.99: 217030.58
------------------------------------------------------
[ 0, 1 ] 68 0.000% 0.000%
( 2900, 4400 ] 89 0.000% 0.000%
( 4400, 6600 ] 33630240 42.038% 42.038% ########
( 6600, 9900 ] 18129842 22.662% 64.700% #####
( 9900, 14000 ] 7877533 9.847% 74.547% ##
( 14000, 22000 ] 15193238 18.992% 93.539% ####
( 22000, 33000 ] 3037061 3.796% 97.335% #
( 33000, 50000 ] 1626316 2.033% 99.368%
( 50000, 75000 ] 421532 0.527% 99.895%
( 75000, 110000 ] 56910 0.071% 99.966%
( 110000, 170000 ] 16134 0.020% 99.986%
( 170000, 250000 ] 5166 0.006% 99.993%
( 250000, 380000 ] 3017 0.004% 99.996%
( 380000, 570000 ] 1337 0.002% 99.998%
( 570000, 860000 ] 805 0.001% 99.999%
( 860000, 1200000 ] 319 0.000% 100.000%
( 1200000, 1900000 ] 231 0.000% 100.000%
( 1900000, 2900000 ] 100 0.000% 100.000%
( 2900000, 4300000 ] 39 0.000% 100.000%
( 4300000, 6500000 ] 16 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
```
New, gather_stats=false. Median thread ops/sec of 5 runs:
```
Complete in 92.030 s; Rough parallel ops/sec = 869285
Thread ops/sec = 54458
Operation latency (ns):
Count: 80000000 Average: 11298.1027 StdDev: 42.18
Min: 0 Median: 7722.0822 Max: 6398720
Percentiles: P50: 7722.08 P75: 14294.68 P99: 47522.95 P99.9: 85292.16 P99.99: 228077.78
------------------------------------------------------
[ 0, 1 ] 109 0.000% 0.000%
( 2900, 4400 ] 793 0.001% 0.001%
( 4400, 6600 ] 34054563 42.568% 42.569% #########
( 6600, 9900 ] 17482646 21.853% 64.423% ####
( 9900, 14000 ] 7908180 9.885% 74.308% ##
( 14000, 22000 ] 15032072 18.790% 93.098% ####
( 22000, 33000 ] 3237834 4.047% 97.145% #
( 33000, 50000 ] 1736882 2.171% 99.316%
( 50000, 75000 ] 446851 0.559% 99.875%
( 75000, 110000 ] 68251 0.085% 99.960%
( 110000, 170000 ] 18592 0.023% 99.983%
( 170000, 250000 ] 7200 0.009% 99.992%
( 250000, 380000 ] 3334 0.004% 99.997%
( 380000, 570000 ] 1393 0.002% 99.998%
( 570000, 860000 ] 700 0.001% 99.999%
( 860000, 1200000 ] 293 0.000% 100.000%
( 1200000, 1900000 ] 196 0.000% 100.000%
( 1900000, 2900000 ] 69 0.000% 100.000%
( 2900000, 4300000 ] 32 0.000% 100.000%
( 4300000, 6500000 ] 10 0.000% 100.000%
```
New, gather_stats=true, 1 second delay between scans. Scans take about
1 second here so it's spending about 50% time scanning. Still the effect on
ops/sec and latency seems to be in the noise. Median thread ops/sec of 5 runs:
```
Complete in 91.890 s; Rough parallel ops/sec = 870608
Thread ops/sec = 54551
Operation latency (ns):
Count: 80000000 Average: 11311.2629 StdDev: 45.28
Min: 0 Median: 7686.5458 Max: 10018340
Percentiles: P50: 7686.55 P75: 14481.95 P99: 47232.60 P99.9: 79230.18 P99.99: 232998.86
------------------------------------------------------
[ 0, 1 ] 71 0.000% 0.000%
( 2900, 4400 ] 291 0.000% 0.000%
( 4400, 6600 ] 34492060 43.115% 43.116% #########
( 6600, 9900 ] 16727328 20.909% 64.025% ####
( 9900, 14000 ] 7845828 9.807% 73.832% ##
( 14000, 22000 ] 15510654 19.388% 93.220% ####
( 22000, 33000 ] 3216533 4.021% 97.241% #
( 33000, 50000 ] 1680859 2.101% 99.342%
( 50000, 75000 ] 439059 0.549% 99.891%
( 75000, 110000 ] 60540 0.076% 99.967%
( 110000, 170000 ] 14649 0.018% 99.985%
( 170000, 250000 ] 5242 0.007% 99.991%
( 250000, 380000 ] 3260 0.004% 99.995%
( 380000, 570000 ] 1599 0.002% 99.997%
( 570000, 860000 ] 1043 0.001% 99.999%
( 860000, 1200000 ] 471 0.001% 99.999%
( 1200000, 1900000 ] 275 0.000% 100.000%
( 1900000, 2900000 ] 143 0.000% 100.000%
( 2900000, 4300000 ] 60 0.000% 100.000%
( 4300000, 6500000 ] 27 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
( 9800000, 14000000 ] 1 0.000% 100.000%
Gather stats latency (us):
Count: 46 Average: 980387.5870 StdDev: 60911.18
Min: 879155 Median: 1033777.7778 Max: 1261431
Percentiles: P50: 1033777.78 P75: 1120666.67 P99: 1261431.00 P99.9: 1261431.00 P99.99: 1261431.00
------------------------------------------------------
( 860000, 1200000 ] 45 97.826% 97.826% ####################
( 1200000, 1900000 ] 1 2.174% 100.000%
Most recent cache entry stats:
Number of entries: 1295133
Total charge: 9.88 GB
Average key size: 23.4982
Average charge: 8.00 KB
Unique deleters: 3
```
Reviewed By: mrambacher
Differential Revision: D28295742
Pulled By: pdillinger
fbshipit-source-id: bbc4a552f91ba0fe10e5cc025c42cef5a81f2b95
2021-05-11 23:16:11 +00:00
|
|
|
*state = UINT32_MAX;
|
|
|
|
} else {
|
2022-06-30 04:50:39 +00:00
|
|
|
*state = index_end << (32 - length_bits);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
New Cache API for gathering statistics (#8225)
Summary:
Adds a new Cache::ApplyToAllEntries API that we expect to use
(in follow-up PRs) for efficiently gathering block cache statistics.
Notable features vs. old ApplyToAllCacheEntries:
* Includes key and deleter (in addition to value and charge). We could
have passed in a Handle but then more virtual function calls would be
needed to get the "fields" of each entry. We expect to use the 'deleter'
to identify the origin of entries, perhaps even more.
* Heavily tuned to minimize latency impact on operating cache. It
does this by iterating over small sections of each cache shard while
cycling through the shards.
* Supports tuning roughly how many entries to operate on for each
lock acquire and release, to control the impact on the latency of other
operations without excessive lock acquire & release. The right balance
can depend on the cost of the callback. Good default seems to be
around 256.
* There should be no need to disable thread safety. (I would expect
uncontended locks to be sufficiently fast.)
I have enhanced cache_bench to validate this approach:
* Reports a histogram of ns per operation, so we can look at the
ditribution of times, not just throughput (average).
* Can add a thread for simulated "gather stats" which calls
ApplyToAllEntries at a specified interval. We also generate a histogram
of time to run ApplyToAllEntries.
To make the iteration over some entries of each shard work as cleanly as
possible, even with resize between next set of entries, I have
re-arranged which hash bits are used for sharding and which for indexing
within a shard.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8225
Test Plan:
A couple of unit tests are added, but primary validation is manual, as
the primary risk is to performance.
The primary validation is using cache_bench to ensure that neither
the minor hashing changes nor the simulated stats gathering
significantly impact QPS or latency distribution. Note that adding op
latency histogram seriously impacts the benchmark QPS, so for a
fair baseline, we need the cache_bench changes (except remove simulated
stat gathering to make it compile). In short, we don't see any
reproducible difference in ops/sec or op latency unless we are gathering
stats nearly continuously. Test uses 10GB block cache with
8KB values to be somewhat realistic in the number of items to iterate
over.
Baseline typical output:
```
Complete in 92.017 s; Rough parallel ops/sec = 869401
Thread ops/sec = 54662
Operation latency (ns):
Count: 80000000 Average: 11223.9494 StdDev: 29.61
Min: 0 Median: 7759.3973 Max: 9620500
Percentiles: P50: 7759.40 P75: 14190.73 P99: 46922.75 P99.9: 77509.84 P99.99: 217030.58
------------------------------------------------------
[ 0, 1 ] 68 0.000% 0.000%
( 2900, 4400 ] 89 0.000% 0.000%
( 4400, 6600 ] 33630240 42.038% 42.038% ########
( 6600, 9900 ] 18129842 22.662% 64.700% #####
( 9900, 14000 ] 7877533 9.847% 74.547% ##
( 14000, 22000 ] 15193238 18.992% 93.539% ####
( 22000, 33000 ] 3037061 3.796% 97.335% #
( 33000, 50000 ] 1626316 2.033% 99.368%
( 50000, 75000 ] 421532 0.527% 99.895%
( 75000, 110000 ] 56910 0.071% 99.966%
( 110000, 170000 ] 16134 0.020% 99.986%
( 170000, 250000 ] 5166 0.006% 99.993%
( 250000, 380000 ] 3017 0.004% 99.996%
( 380000, 570000 ] 1337 0.002% 99.998%
( 570000, 860000 ] 805 0.001% 99.999%
( 860000, 1200000 ] 319 0.000% 100.000%
( 1200000, 1900000 ] 231 0.000% 100.000%
( 1900000, 2900000 ] 100 0.000% 100.000%
( 2900000, 4300000 ] 39 0.000% 100.000%
( 4300000, 6500000 ] 16 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
```
New, gather_stats=false. Median thread ops/sec of 5 runs:
```
Complete in 92.030 s; Rough parallel ops/sec = 869285
Thread ops/sec = 54458
Operation latency (ns):
Count: 80000000 Average: 11298.1027 StdDev: 42.18
Min: 0 Median: 7722.0822 Max: 6398720
Percentiles: P50: 7722.08 P75: 14294.68 P99: 47522.95 P99.9: 85292.16 P99.99: 228077.78
------------------------------------------------------
[ 0, 1 ] 109 0.000% 0.000%
( 2900, 4400 ] 793 0.001% 0.001%
( 4400, 6600 ] 34054563 42.568% 42.569% #########
( 6600, 9900 ] 17482646 21.853% 64.423% ####
( 9900, 14000 ] 7908180 9.885% 74.308% ##
( 14000, 22000 ] 15032072 18.790% 93.098% ####
( 22000, 33000 ] 3237834 4.047% 97.145% #
( 33000, 50000 ] 1736882 2.171% 99.316%
( 50000, 75000 ] 446851 0.559% 99.875%
( 75000, 110000 ] 68251 0.085% 99.960%
( 110000, 170000 ] 18592 0.023% 99.983%
( 170000, 250000 ] 7200 0.009% 99.992%
( 250000, 380000 ] 3334 0.004% 99.997%
( 380000, 570000 ] 1393 0.002% 99.998%
( 570000, 860000 ] 700 0.001% 99.999%
( 860000, 1200000 ] 293 0.000% 100.000%
( 1200000, 1900000 ] 196 0.000% 100.000%
( 1900000, 2900000 ] 69 0.000% 100.000%
( 2900000, 4300000 ] 32 0.000% 100.000%
( 4300000, 6500000 ] 10 0.000% 100.000%
```
New, gather_stats=true, 1 second delay between scans. Scans take about
1 second here so it's spending about 50% time scanning. Still the effect on
ops/sec and latency seems to be in the noise. Median thread ops/sec of 5 runs:
```
Complete in 91.890 s; Rough parallel ops/sec = 870608
Thread ops/sec = 54551
Operation latency (ns):
Count: 80000000 Average: 11311.2629 StdDev: 45.28
Min: 0 Median: 7686.5458 Max: 10018340
Percentiles: P50: 7686.55 P75: 14481.95 P99: 47232.60 P99.9: 79230.18 P99.99: 232998.86
------------------------------------------------------
[ 0, 1 ] 71 0.000% 0.000%
( 2900, 4400 ] 291 0.000% 0.000%
( 4400, 6600 ] 34492060 43.115% 43.116% #########
( 6600, 9900 ] 16727328 20.909% 64.025% ####
( 9900, 14000 ] 7845828 9.807% 73.832% ##
( 14000, 22000 ] 15510654 19.388% 93.220% ####
( 22000, 33000 ] 3216533 4.021% 97.241% #
( 33000, 50000 ] 1680859 2.101% 99.342%
( 50000, 75000 ] 439059 0.549% 99.891%
( 75000, 110000 ] 60540 0.076% 99.967%
( 110000, 170000 ] 14649 0.018% 99.985%
( 170000, 250000 ] 5242 0.007% 99.991%
( 250000, 380000 ] 3260 0.004% 99.995%
( 380000, 570000 ] 1599 0.002% 99.997%
( 570000, 860000 ] 1043 0.001% 99.999%
( 860000, 1200000 ] 471 0.001% 99.999%
( 1200000, 1900000 ] 275 0.000% 100.000%
( 1900000, 2900000 ] 143 0.000% 100.000%
( 2900000, 4300000 ] 60 0.000% 100.000%
( 4300000, 6500000 ] 27 0.000% 100.000%
( 6500000, 9800000 ] 7 0.000% 100.000%
( 9800000, 14000000 ] 1 0.000% 100.000%
Gather stats latency (us):
Count: 46 Average: 980387.5870 StdDev: 60911.18
Min: 879155 Median: 1033777.7778 Max: 1261431
Percentiles: P50: 1033777.78 P75: 1120666.67 P99: 1261431.00 P99.9: 1261431.00 P99.99: 1261431.00
------------------------------------------------------
( 860000, 1200000 ] 45 97.826% 97.826% ####################
( 1200000, 1900000 ] 1 2.174% 100.000%
Most recent cache entry stats:
Number of entries: 1295133
Total charge: 9.88 GB
Average key size: 23.4982
Average charge: 8.00 KB
Unique deleters: 3
```
Reviewed By: mrambacher
Differential Revision: D28295742
Pulled By: pdillinger
fbshipit-source-id: bbc4a552f91ba0fe10e5cc025c42cef5a81f2b95
2021-05-11 23:16:11 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
table_.ApplyToEntriesRange(
|
|
|
|
[callback,
|
|
|
|
metadata_charge_policy = metadata_charge_policy_](ClockHandle* h) {
|
|
|
|
callback(h->key(), h->value, h->GetCharge(metadata_charge_policy),
|
|
|
|
h->deleter);
|
|
|
|
},
|
2022-07-16 05:36:58 +00:00
|
|
|
index_begin, index_end, false);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-07-28 01:55:55 +00:00
|
|
|
ClockHandle* ClockCacheShard::DetachedInsert(ClockHandle* h) {
|
|
|
|
ClockHandle* e = new ClockHandle();
|
|
|
|
*e = *h;
|
|
|
|
e->SetDetached();
|
|
|
|
e->TryExternalRef();
|
|
|
|
detached_usage_ += h->total_charge;
|
|
|
|
return e;
|
|
|
|
}
|
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
size_t ClockCacheShard::CalcEstimatedHandleCharge(
|
|
|
|
size_t estimated_value_size,
|
|
|
|
CacheMetadataChargePolicy metadata_charge_policy) {
|
|
|
|
ClockHandle h;
|
|
|
|
h.CalcTotalCharge(estimated_value_size, metadata_charge_policy);
|
|
|
|
return h.total_charge;
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
int ClockCacheShard::CalcHashBits(
|
|
|
|
size_t capacity, size_t estimated_value_size,
|
|
|
|
CacheMetadataChargePolicy metadata_charge_policy) {
|
|
|
|
size_t handle_charge =
|
|
|
|
CalcEstimatedHandleCharge(estimated_value_size, metadata_charge_policy);
|
2022-07-02 03:51:20 +00:00
|
|
|
assert(handle_charge > 0);
|
2022-06-30 04:50:39 +00:00
|
|
|
uint32_t num_entries =
|
2022-07-02 03:51:20 +00:00
|
|
|
static_cast<uint32_t>(capacity / (kLoadFactor * handle_charge)) + 1;
|
|
|
|
assert(num_entries <= uint32_t{1} << 31);
|
|
|
|
return FloorLog2((num_entries << 1) - 1);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-07-27 00:42:03 +00:00
|
|
|
void ClockCacheShard::SetCapacity(size_t capacity) {
|
|
|
|
if (capacity > table_.GetCapacity()) {
|
|
|
|
assert(false); // Not supported.
|
|
|
|
}
|
|
|
|
table_.SetCapacity(capacity);
|
|
|
|
table_.ClockRun(detached_usage_);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-07-27 00:42:03 +00:00
|
|
|
void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
|
|
|
|
strict_capacity_limit_ = strict_capacity_limit;
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
|
2022-06-30 04:50:39 +00:00
|
|
|
size_t charge, Cache::DeleterFn deleter,
|
|
|
|
Cache::Handle** handle,
|
2022-07-07 01:28:35 +00:00
|
|
|
Cache::Priority priority) {
|
2022-06-30 04:50:39 +00:00
|
|
|
if (key.size() != kCacheKeySize) {
|
|
|
|
return Status::NotSupported("ClockCache only supports key size " +
|
|
|
|
std::to_string(kCacheKeySize) + "B");
|
|
|
|
}
|
|
|
|
|
|
|
|
ClockHandle tmp;
|
|
|
|
tmp.value = value;
|
|
|
|
tmp.deleter = deleter;
|
|
|
|
tmp.hash = hash;
|
|
|
|
tmp.CalcTotalCharge(charge, metadata_charge_policy_);
|
2022-07-07 01:28:35 +00:00
|
|
|
tmp.SetCachePriority(priority);
|
2022-06-30 04:50:39 +00:00
|
|
|
for (int i = 0; i < kCacheKeySize; i++) {
|
|
|
|
tmp.key_data[i] = key.data()[i];
|
|
|
|
}
|
|
|
|
|
|
|
|
Status s = Status::OK();
|
2022-07-25 17:02:19 +00:00
|
|
|
|
2022-07-27 00:42:03 +00:00
|
|
|
// Use a local copy to minimize cache synchronization.
|
|
|
|
size_t detached_usage = detached_usage_;
|
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
// Free space with the clock policy until enough space is freed or there are
|
|
|
|
// no evictable elements.
|
2022-07-27 00:42:03 +00:00
|
|
|
table_.ClockRun(tmp.total_charge + detached_usage);
|
2022-07-25 17:02:19 +00:00
|
|
|
|
2022-07-27 00:42:03 +00:00
|
|
|
// Use local copies to minimize cache synchronization
|
|
|
|
// (occupancy_ and usage_ are read and written by all insertions).
|
2022-07-25 17:02:19 +00:00
|
|
|
uint32_t occupancy_local = table_.GetOccupancy();
|
2022-07-27 00:42:03 +00:00
|
|
|
size_t total_usage = table_.GetUsage() + detached_usage;
|
|
|
|
|
|
|
|
// TODO: Currently we support strict_capacity_limit == false as long as the
|
|
|
|
// number of pinned elements is below table_.GetOccupancyLimit(). We can
|
|
|
|
// always support it as follows: whenever we exceed this limit, we dynamically
|
|
|
|
// allocate a handle and return it (when the user provides a handle pointer,
|
|
|
|
// of course). Then, Release checks whether the handle was dynamically
|
|
|
|
// allocated, or is stored in the table.
|
|
|
|
if (total_usage + tmp.total_charge > table_.GetCapacity() &&
|
|
|
|
(strict_capacity_limit_ || handle == nullptr)) {
|
2022-07-25 17:02:19 +00:00
|
|
|
if (handle == nullptr) {
|
|
|
|
// Don't insert the entry but still return ok, as if the entry inserted
|
|
|
|
// into cache and get evicted immediately.
|
2022-07-27 00:42:03 +00:00
|
|
|
tmp.FreeData();
|
2016-08-23 20:53:49 +00:00
|
|
|
} else {
|
2022-07-27 00:42:03 +00:00
|
|
|
if (occupancy_local + 1 > table_.GetOccupancyLimit()) {
|
2022-07-25 17:02:19 +00:00
|
|
|
// TODO: Consider using a distinct status for this case, but usually
|
|
|
|
// it will be handled the same way as reaching charge capacity limit
|
|
|
|
s = Status::MemoryLimit(
|
|
|
|
"Insert failed because all slots in the hash table are full.");
|
2022-06-30 04:50:39 +00:00
|
|
|
} else {
|
2022-07-25 17:02:19 +00:00
|
|
|
s = Status::MemoryLimit(
|
|
|
|
"Insert failed because the total charge has exceeded the "
|
|
|
|
"capacity.");
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-23 20:53:49 +00:00
|
|
|
}
|
2022-07-25 17:02:19 +00:00
|
|
|
} else {
|
2022-07-28 01:55:55 +00:00
|
|
|
ClockHandle* h = nullptr;
|
|
|
|
if (handle != nullptr && occupancy_local + 1 > table_.GetOccupancyLimit()) {
|
2022-07-27 00:42:03 +00:00
|
|
|
// Even if the user wishes to overload the cache, we can't insert into
|
|
|
|
// the hash table. Instead, we dynamically allocate a new handle.
|
2022-07-28 01:55:55 +00:00
|
|
|
h = DetachedInsert(&tmp);
|
2022-07-27 00:42:03 +00:00
|
|
|
// TODO: Return special status?
|
|
|
|
} else {
|
|
|
|
// Insert into the cache. Note that the cache might get larger than its
|
|
|
|
// capacity if not enough space was freed up.
|
|
|
|
autovector<ClockHandle> deleted;
|
|
|
|
h = table_.Insert(&tmp, &deleted, handle != nullptr);
|
2022-07-28 01:55:55 +00:00
|
|
|
if (h == nullptr && handle != nullptr) {
|
|
|
|
// The table is full. This can happen when many threads simultaneously
|
|
|
|
// attempt an insert, and the table is operating close to full capacity.
|
|
|
|
h = DetachedInsert(&tmp);
|
|
|
|
}
|
|
|
|
// Notice that if handle == nullptr, we don't insert the entry but still
|
|
|
|
// return ok.
|
2022-07-27 00:42:03 +00:00
|
|
|
if (deleted.size() > 0) {
|
|
|
|
s = Status::OkOverwritten();
|
|
|
|
}
|
|
|
|
table_.Free(&deleted);
|
|
|
|
}
|
2022-07-25 17:02:19 +00:00
|
|
|
if (handle != nullptr) {
|
|
|
|
*handle = reinterpret_cast<Cache::Handle*>(h);
|
|
|
|
}
|
2020-04-27 20:18:18 +00:00
|
|
|
}
|
2022-06-30 04:50:39 +00:00
|
|
|
|
2016-08-19 19:28:19 +00:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2022-07-16 05:36:58 +00:00
|
|
|
Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
|
2022-07-25 17:02:19 +00:00
|
|
|
return reinterpret_cast<Cache::Handle*>(table_.Lookup(key, hash));
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
bool ClockCacheShard::Ref(Cache::Handle* h) {
|
|
|
|
ClockHandle* e = reinterpret_cast<ClockHandle*>(h);
|
2022-07-27 00:42:03 +00:00
|
|
|
assert(e->ExternalRefs() > 0);
|
2022-07-16 05:36:58 +00:00
|
|
|
return e->TryExternalRef();
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
bool ClockCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
|
2022-07-16 05:36:58 +00:00
|
|
|
// In contrast with LRUCache's Release, this function won't delete the handle
|
2022-07-25 17:02:19 +00:00
|
|
|
// when the cache is above capacity and the reference is the last one. Space
|
2022-07-16 05:36:58 +00:00
|
|
|
// is only freed up by EvictFromClock (called by Insert when space is needed)
|
2022-07-25 17:02:19 +00:00
|
|
|
// and Erase. We do this to avoid an extra atomic read of the variable usage_.
|
2022-06-30 04:50:39 +00:00
|
|
|
if (handle == nullptr) {
|
|
|
|
return false;
|
|
|
|
}
|
2022-07-16 05:36:58 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
ClockHandle* h = reinterpret_cast<ClockHandle*>(handle);
|
2022-07-27 00:42:03 +00:00
|
|
|
|
|
|
|
if (UNLIKELY(h->IsDetached())) {
|
|
|
|
h->ReleaseExternalRef();
|
|
|
|
if (h->TryExclusiveRef()) {
|
|
|
|
// Only the last reference will succeed.
|
|
|
|
// Don't bother releasing the exclusive ref.
|
|
|
|
h->FreeData();
|
|
|
|
detached_usage_ -= h->total_charge;
|
|
|
|
delete h;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
uint32_t refs = h->refs;
|
|
|
|
bool last_reference = ((refs & ClockHandle::EXTERNAL_REFS) == 1);
|
2022-07-16 05:36:58 +00:00
|
|
|
bool will_be_deleted = refs & ClockHandle::WILL_BE_DELETED;
|
|
|
|
|
|
|
|
if (last_reference && (will_be_deleted || erase_if_last_ref)) {
|
2022-07-25 17:02:19 +00:00
|
|
|
autovector<ClockHandle> deleted;
|
|
|
|
h->SetWillBeDeleted(true);
|
|
|
|
h->ReleaseExternalRef();
|
|
|
|
if (table_.SpinTryRemove(h, &deleted)) {
|
|
|
|
h->ReleaseExclusiveRef();
|
|
|
|
table_.Free(&deleted);
|
|
|
|
return true;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2022-07-25 17:02:19 +00:00
|
|
|
} else {
|
|
|
|
h->ReleaseExternalRef();
|
2017-04-24 18:21:47 +00:00
|
|
|
}
|
2022-07-16 05:36:58 +00:00
|
|
|
|
|
|
|
return false;
|
2017-04-24 18:21:47 +00:00
|
|
|
}
|
|
|
|
|
2022-07-16 05:36:58 +00:00
|
|
|
void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
|
2022-07-25 17:02:19 +00:00
|
|
|
autovector<ClockHandle> deleted;
|
|
|
|
uint32_t probe = 0;
|
|
|
|
table_.RemoveAll(key, hash, probe, &deleted);
|
|
|
|
table_.Free(&deleted);
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-07-25 17:02:19 +00:00
|
|
|
size_t ClockCacheShard::GetUsage() const { return table_.GetUsage(); }
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
size_t ClockCacheShard::GetPinnedUsage() const {
|
2022-07-25 17:02:19 +00:00
|
|
|
// Computes the pinned usage by scanning the whole hash table. This
|
|
|
|
// is slow, but avoids keeping an exact counter on the clock usage,
|
2022-07-16 05:36:58 +00:00
|
|
|
// i.e., the number of not externally referenced elements.
|
2022-07-25 17:02:19 +00:00
|
|
|
// Why avoid this counter? Because Lookup removes elements from the clock
|
2022-07-16 05:36:58 +00:00
|
|
|
// list, so it would need to update the pinned usage every time,
|
|
|
|
// which creates additional synchronization costs.
|
|
|
|
size_t clock_usage = 0;
|
|
|
|
|
|
|
|
table_.ConstApplyToEntriesRange(
|
|
|
|
[&clock_usage](ClockHandle* h) {
|
2022-07-27 00:42:03 +00:00
|
|
|
if (h->ExternalRefs() > 1) {
|
|
|
|
// We check > 1 because we are holding an external ref.
|
2022-07-16 05:36:58 +00:00
|
|
|
clock_usage += h->total_charge;
|
|
|
|
}
|
|
|
|
},
|
|
|
|
0, table_.GetTableSize(), true);
|
|
|
|
|
2022-07-27 00:42:03 +00:00
|
|
|
return clock_usage + detached_usage_;
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
ClockCache::ClockCache(size_t capacity, size_t estimated_value_size,
|
|
|
|
int num_shard_bits, bool strict_capacity_limit,
|
|
|
|
CacheMetadataChargePolicy metadata_charge_policy)
|
2022-07-25 17:02:19 +00:00
|
|
|
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit),
|
|
|
|
num_shards_(1 << num_shard_bits) {
|
2022-07-02 03:51:20 +00:00
|
|
|
assert(estimated_value_size > 0 ||
|
|
|
|
metadata_charge_policy != kDontChargeCacheMetadata);
|
2022-06-30 04:50:39 +00:00
|
|
|
shards_ = reinterpret_cast<ClockCacheShard*>(
|
|
|
|
port::cacheline_aligned_alloc(sizeof(ClockCacheShard) * num_shards_));
|
|
|
|
size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
|
|
|
|
for (int i = 0; i < num_shards_; i++) {
|
|
|
|
new (&shards_[i])
|
|
|
|
ClockCacheShard(per_shard, estimated_value_size, strict_capacity_limit,
|
|
|
|
metadata_charge_policy);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
ClockCache::~ClockCache() {
|
|
|
|
if (shards_ != nullptr) {
|
|
|
|
assert(num_shards_ > 0);
|
|
|
|
for (int i = 0; i < num_shards_; i++) {
|
|
|
|
shards_[i].~ClockCacheShard();
|
|
|
|
}
|
|
|
|
port::cacheline_aligned_free(shards_);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
2022-06-30 04:50:39 +00:00
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
CacheShard* ClockCache::GetShard(uint32_t shard) {
|
|
|
|
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
const CacheShard* ClockCache::GetShard(uint32_t shard) const {
|
|
|
|
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
void* ClockCache::Value(Handle* handle) {
|
|
|
|
return reinterpret_cast<const ClockHandle*>(handle)->value;
|
|
|
|
}
|
Use deleters to label cache entries and collect stats (#8297)
Summary:
This change gathers and publishes statistics about the
kinds of items in block cache. This is especially important for
profiling relative usage of cache by index vs. filter vs. data blocks.
It works by iterating over the cache during periodic stats dump
(InternalStats, stats_dump_period_sec) or on demand when
DB::Get(Map)Property(kBlockCacheEntryStats), except that for
efficiency and sharing among column families, saved data from
the last scan is used when the data is not considered too old.
The new information can be seen in info LOG, for example:
Block cache LRUCache@0x7fca62229330 capacity: 95.37 MB collections: 8 last_copies: 0 last_secs: 0.00178 secs_since: 0
Block cache entry stats(count,size,portion): DataBlock(7092,28.24 MB,29.6136%) FilterBlock(215,867.90 KB,0.888728%) FilterMetaBlock(2,5.31 KB,0.00544%) IndexBlock(217,180.11 KB,0.184432%) WriteBuffer(1,256.00 KB,0.262144%) Misc(1,0.00 KB,0%)
And also through DB::GetProperty and GetMapProperty (here using
ldb just for demonstration):
$ ./ldb --db=/dev/shm/dbbench/ get_property rocksdb.block-cache-entry-stats
rocksdb.block-cache-entry-stats.bytes.data-block: 0
rocksdb.block-cache-entry-stats.bytes.deprecated-filter-block: 0
rocksdb.block-cache-entry-stats.bytes.filter-block: 0
rocksdb.block-cache-entry-stats.bytes.filter-meta-block: 0
rocksdb.block-cache-entry-stats.bytes.index-block: 178992
rocksdb.block-cache-entry-stats.bytes.misc: 0
rocksdb.block-cache-entry-stats.bytes.other-block: 0
rocksdb.block-cache-entry-stats.bytes.write-buffer: 0
rocksdb.block-cache-entry-stats.capacity: 8388608
rocksdb.block-cache-entry-stats.count.data-block: 0
rocksdb.block-cache-entry-stats.count.deprecated-filter-block: 0
rocksdb.block-cache-entry-stats.count.filter-block: 0
rocksdb.block-cache-entry-stats.count.filter-meta-block: 0
rocksdb.block-cache-entry-stats.count.index-block: 215
rocksdb.block-cache-entry-stats.count.misc: 1
rocksdb.block-cache-entry-stats.count.other-block: 0
rocksdb.block-cache-entry-stats.count.write-buffer: 0
rocksdb.block-cache-entry-stats.id: LRUCache@0x7f3636661290
rocksdb.block-cache-entry-stats.percent.data-block: 0.000000
rocksdb.block-cache-entry-stats.percent.deprecated-filter-block: 0.000000
rocksdb.block-cache-entry-stats.percent.filter-block: 0.000000
rocksdb.block-cache-entry-stats.percent.filter-meta-block: 0.000000
rocksdb.block-cache-entry-stats.percent.index-block: 2.133751
rocksdb.block-cache-entry-stats.percent.misc: 0.000000
rocksdb.block-cache-entry-stats.percent.other-block: 0.000000
rocksdb.block-cache-entry-stats.percent.write-buffer: 0.000000
rocksdb.block-cache-entry-stats.secs_for_last_collection: 0.000052
rocksdb.block-cache-entry-stats.secs_since_last_collection: 0
Solution detail - We need some way to flag what kind of blocks each
entry belongs to, preferably without changing the Cache API.
One of the complications is that Cache is a general interface that could
have other users that don't adhere to whichever convention we decide
on for keys and values. Or we would pay for an extra field in the Handle
that would only be used for this purpose.
This change uses a back-door approach, the deleter, to indicate the
"role" of a Cache entry (in addition to the value type, implicitly).
This has the added benefit of ensuring proper code origin whenever we
recognize a particular role for a cache entry; if the entry came from
some other part of the code, it will use an unrecognized deleter, which
we simply attribute to the "Misc" role.
An internal API makes for simple instantiation and automatic
registration of Cache deleters for a given value type and "role".
Another internal API, CacheEntryStatsCollector, solves the problem of
caching the results of a scan and sharing them, to ensure scans are
neither excessive nor redundant so as not to harm Cache performance.
Because code is added to BlocklikeTraits, it is pulled out of
block_based_table_reader.cc into its own file.
This is a reformulation of https://github.com/facebook/rocksdb/issues/8276, without the type checking option
(could still be added), and with actual stat gathering.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8297
Test Plan: manual testing with db_bench, and a couple of basic unit tests
Reviewed By: ltamasi
Differential Revision: D28488721
Pulled By: pdillinger
fbshipit-source-id: 472f524a9691b5afb107934be2d41d84f2b129fb
2021-05-19 23:45:51 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
size_t ClockCache::GetCharge(Handle* handle) const {
|
|
|
|
CacheMetadataChargePolicy metadata_charge_policy = kDontChargeCacheMetadata;
|
|
|
|
if (num_shards_ > 0) {
|
|
|
|
metadata_charge_policy = shards_[0].metadata_charge_policy_;
|
Fix use-after-free threading bug in ClockCache (#8261)
Summary:
In testing for https://github.com/facebook/rocksdb/issues/8225 I found cache_bench would crash with
-use_clock_cache, as well as db_bench -use_clock_cache, but not
single-threaded. Smaller cache size hits failure much faster. ASAN
reported the failuer as calling malloc_usable_size on the `key` pointer
of a ClockCache handle after it was reportedly freed. On detailed
inspection I found this bad sequence of operations for a cache entry:
state=InCache=1,refs=1
[thread 1] Start ClockCacheShard::Unref (from Release, no mutex)
[thread 1] Decrement ref count
state=InCache=1,refs=0
[thread 1] Suspend before CalcTotalCharge (no mutex)
[thread 2] Start UnsetInCache (from Insert, mutex held)
[thread 2] clear InCache bit
state=InCache=0,refs=0
[thread 2] Calls RecycleHandle (based on pre-updated state)
[thread 2] Returns to Insert which calls Cleanup which deletes `key`
[thread 1] Resume ClockCacheShard::Unref
[thread 1] Read `key` in CalcTotalCharge
To fix this, I've added a field to the handle to store the metadata
charge so that we can efficiently remember everything we need from
the handle in Unref. We must not read from the handle again if we
decrement the count to zero with InCache=1, which means we don't own
the entry and someone else could eject/overwrite it immediately.
Note before this change, on amd64 sizeof(Handle) == 56 even though there
are only 48 bytes of data. Grouping together the uint32_t fields would
cut it down to 48, but I've added another uint32_t, which takes it
back up to 56. Not a big deal.
Also fixed DisownData to cooperate with ASAN as in LRUCache.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8261
Test Plan:
Manual + adding use_clock_cache to db_crashtest.py
Base performance
./cache_bench -use_clock_cache
Complete in 17.060 s; QPS = 2458513
New performance
./cache_bench -use_clock_cache
Complete in 17.052 s; QPS = 2459695
Any difference is easily buried in small noise.
Crash test shows still more bug(s) in ClockCache, so I'm expecting to
disable ClockCache from production code in a follow-up PR (if we
can't find and fix the bug(s))
Reviewed By: mrambacher
Differential Revision: D28207358
Pulled By: pdillinger
fbshipit-source-id: aa7a9322afc6f18f30e462c75dbbe4a1206eb294
2021-05-05 05:17:02 +00:00
|
|
|
}
|
2022-06-30 04:50:39 +00:00
|
|
|
return reinterpret_cast<const ClockHandle*>(handle)->GetCharge(
|
|
|
|
metadata_charge_policy);
|
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
Cache::DeleterFn ClockCache::GetDeleter(Handle* handle) const {
|
|
|
|
auto h = reinterpret_cast<const ClockHandle*>(handle);
|
|
|
|
return h->deleter;
|
|
|
|
}
|
2021-05-14 05:57:51 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
uint32_t ClockCache::GetHash(Handle* handle) const {
|
|
|
|
return reinterpret_cast<const ClockHandle*>(handle)->hash;
|
|
|
|
}
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2022-06-30 04:50:39 +00:00
|
|
|
void ClockCache::DisownData() {
|
|
|
|
// Leak data only if that won't generate an ASAN/valgrind warning.
|
|
|
|
if (!kMustFreeHeapAllocations) {
|
|
|
|
shards_ = nullptr;
|
|
|
|
num_shards_ = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace clock_cache
|
2016-08-19 19:28:19 +00:00
|
|
|
|
2019-09-16 22:14:51 +00:00
|
|
|
std::shared_ptr<Cache> NewClockCache(
|
2022-07-14 00:43:39 +00:00
|
|
|
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
2022-07-13 15:45:44 +00:00
|
|
|
CacheMetadataChargePolicy metadata_charge_policy) {
|
2022-07-29 14:18:15 +00:00
|
|
|
return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit, 0.5,
|
|
|
|
nullptr, kDefaultToAdaptiveMutex, metadata_charge_policy);
|
2022-07-13 15:45:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
std::shared_ptr<Cache> ExperimentalNewClockCache(
|
2022-06-30 04:50:39 +00:00
|
|
|
size_t capacity, size_t estimated_value_size, int num_shard_bits,
|
|
|
|
bool strict_capacity_limit,
|
2019-09-16 22:14:51 +00:00
|
|
|
CacheMetadataChargePolicy metadata_charge_policy) {
|
2022-06-30 04:50:39 +00:00
|
|
|
if (num_shard_bits >= 20) {
|
|
|
|
return nullptr; // The cache cannot be sharded into too many fine pieces.
|
|
|
|
}
|
2017-01-27 14:35:41 +00:00
|
|
|
if (num_shard_bits < 0) {
|
|
|
|
num_shard_bits = GetDefaultCacheShardBits(capacity);
|
|
|
|
}
|
2022-06-30 04:50:39 +00:00
|
|
|
return std::make_shared<clock_cache::ClockCache>(
|
|
|
|
capacity, estimated_value_size, num_shard_bits, strict_capacity_limit,
|
|
|
|
metadata_charge_policy);
|
2016-08-19 19:28:19 +00:00
|
|
|
}
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|