dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate some variants of Get and MultiGet (#12327) 

Summary:
A lot of variants of Get and MultiGet have been added to `include/rocksdb/db.h` over the years. Try to consolidate them by marking variants that don't return timestamps as deprecated. The underlying DB implementation will check and return Status::NotSupported() if it doesn't support returning timestamps and the caller asks for it.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/12327

Reviewed By: pdillinger

Differential Revision: D53828151

Pulled By: anand1976

fbshipit-source-id: e0b5ca42d32daa2739d5f439a729815a2d4ff050
This commit is contained in:
anand76 2024-02-16 09:21:06 -08:00 committed by Facebook GitHub Bot
parent 956f1dfde3
commit d227276147
19 changed files with 346 additions and 702 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
db/db_basic_test.cc
View File

@ -4504,68 +4504,35 @@ TEST_P(DBBasicTestMultiGetDeadline, MultiGetDeadlineExceeded) {
SetTimeElapseOnlySleepOnReopen(&options);
ReopenWithColumnFamilies(GetCFNames(), options);
// Test the non-batched version of MultiGet with multiple column
// families
// Test batched MultiGet with an IO delay in the first data block read.
// Both keys in the first CF should succeed as they're in the same data
// block and would form one batch, and we check for deadline between
// batches.
std::vector<std::string> key_str;
size_t i;
for (i = 0; i < 5; ++i) {
for (i = 0; i < 10; ++i) {
key_str.emplace_back(Key(static_cast<int>(i)));
}
std::vector<ColumnFamilyHandle*> cfs(key_str.size());
;
std::vector<Slice> keys(key_str.size());
std::vector<std::string> values(key_str.size());
std::vector<PinnableSlice> pin_values(keys.size());
for (i = 0; i < key_str.size(); ++i) {
cfs[i] = handles_[i];
// 2 keys per CF
cfs[i] = handles_[i / 2];
keys[i] = Slice(key_str[i].data(), key_str[i].size());
}
ReadOptions ro;
ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000};
ro.async_io = GetParam();
// Delay the first IO
fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 0);
std::vector<Status> statuses = dbfull()->MultiGet(ro, cfs, keys, &values);
// The first key is successful because we check after the lookup, but
// subsequent keys fail due to deadline exceeded
CheckStatus(statuses, 1);
// Clear the cache
cache->SetCapacity(0);
cache->SetCapacity(1048576);
// Test non-batched Multiget with multiple column families and
// introducing an IO delay in one of the middle CFs
key_str.clear();
for (i = 0; i < 10; ++i) {
key_str.emplace_back(Key(static_cast<int>(i)));
}
cfs.resize(key_str.size());
keys.resize(key_str.size());
values.resize(key_str.size());
for (i = 0; i < key_str.size(); ++i) {
// 2 keys per CF
cfs[i] = handles_[i / 2];
keys[i] = Slice(key_str[i].data(), key_str[i].size());
}
ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000};
fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 1);
statuses = dbfull()->MultiGet(ro, cfs, keys, &values);
CheckStatus(statuses, 3);
// Test batched MultiGet with an IO delay in the first data block read.
// Both keys in the first CF should succeed as they're in the same data
// block and would form one batch, and we check for deadline between
// batches.
std::vector<PinnableSlice> pin_values(keys.size());
cache->SetCapacity(0);
cache->SetCapacity(1048576);
statuses.clear();
statuses.resize(keys.size());
ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000};
fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 0);
std::vector<Status> statuses(key_str.size());
dbfull()->MultiGet(ro, keys.size(), cfs.data(), keys.data(),
pin_values.data(), statuses.data());
// The first key is successful because we check after the lookup, but
// subsequent keys fail due to deadline exceeded
CheckStatus(statuses, 2);
// Similar to the previous one, but an IO delay in the third CF data block

94
db/db_impl/compacted_db_impl.cc
View File

@ -33,12 +33,6 @@ size_t CompactedDBImpl::FindFile(const Slice& key) {
files_.files);
}
Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
const Slice& key, PinnableSlice* value) {
return Get(options, /*column_family*/ nullptr, key, value,
/*timestamp*/ nullptr);
}
Status CompactedDBImpl::Get(const ReadOptions& _read_options,
ColumnFamilyHandle*, const Slice& key,
PinnableSlice* value, std::string* timestamp) {
@ -108,62 +102,59 @@ Status CompactedDBImpl::Get(const ReadOptions& _read_options,
return Status::NotFound();
}
std::vector<Status> CompactedDBImpl::MultiGet(
const ReadOptions& options, const std::vector<ColumnFamilyHandle*>&,
const std::vector<Slice>& keys, std::vector<std::string>* values) {
return MultiGet(options, keys, values, /*timestamps*/ nullptr);
}
std::vector<Status> CompactedDBImpl::MultiGet(
const ReadOptions& _read_options, const std::vector<ColumnFamilyHandle*>&,
const std::vector<Slice>& keys, std::vector<std::string>* values,
std::vector<std::string>* timestamps) {
void CompactedDBImpl::MultiGet(const ReadOptions& _read_options,
size_t num_keys,
ColumnFamilyHandle** /*column_families*/,
const Slice* keys, PinnableSlice* values,
std::string* timestamps, Status* statuses,
const bool /*sorted_input*/) {
assert(user_comparator_);
size_t num_keys = keys.size();
Status s;
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kMultiGet) {
Status s = Status::InvalidArgument(
s = Status::InvalidArgument(
"Can only call MultiGet with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
return std::vector<Status>(num_keys, s);
}
ReadOptions read_options(_read_options);
if (read_options.io_activity == Env::IOActivity::kUnknown) {
read_options.io_activity = Env::IOActivity::kMultiGet;
if (s.ok()) {
if (read_options.io_activity == Env::IOActivity::kUnknown) {
read_options.io_activity = Env::IOActivity::kMultiGet;
}
if (read_options.timestamp) {
s = FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
if (s.ok()) {
if (read_options.timestamp->size() > 0) {
s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
*(read_options.timestamp));
}
}
} else {
s = FailIfCfHasTs(DefaultColumnFamily());
}
}
if (read_options.timestamp) {
Status s =
FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
if (!s.ok()) {
return std::vector<Status>(num_keys, s);
}
if (read_options.timestamp->size() > 0) {
s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
*(read_options.timestamp));
if (!s.ok()) {
return std::vector<Status>(num_keys, s);
}
}
} else {
Status s = FailIfCfHasTs(DefaultColumnFamily());
if (!s.ok()) {
return std::vector<Status>(num_keys, s);
if (!s.ok()) {
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] = s;
}
return;
}
// Clear the timestamps for returning results so that we can distinguish
// between tombstone or key that has never been written
if (timestamps) {
for (auto& ts : *timestamps) {
ts.clear();
for (size_t i = 0; i < num_keys; ++i) {
timestamps[i].clear();
}
}
GetWithTimestampReadCallback read_cb(kMaxSequenceNumber);
autovector<TableReader*, 16> reader_list;
for (const auto& key : keys) {
for (size_t i = 0; i < num_keys; ++i) {
const Slice& key = keys[i];
LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp);
const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())];
if (user_comparator_->CompareWithoutTimestamp(
@ -177,30 +168,26 @@ std::vector<Status> CompactedDBImpl::MultiGet(
reader_list.push_back(f.fd.table_reader);
}
}
std::vector<Status> statuses(num_keys, Status::NotFound());
values->resize(num_keys);
if (timestamps) {
timestamps->resize(num_keys);
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] = Status::NotFound();
}
int idx = 0;
for (auto* r : reader_list) {
if (r != nullptr) {
PinnableSlice pinnable_val;
std::string& value = (*values)[idx];
PinnableSlice& pinnable_val = values[idx];
LookupKey lkey(keys[idx], kMaxSequenceNumber, read_options.timestamp);
std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr;
std::string* timestamp = timestamps ? &timestamps[idx] : nullptr;
GetContext get_context(
user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound,
lkey.user_key(), &pinnable_val, /*columns=*/nullptr,
user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr,
nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb);
Status s =
Status status =
r->Get(read_options, lkey.internal_key(), &get_context, nullptr);
assert(static_cast<size_t>(idx) < statuses.size());
if (!s.ok() && !s.IsNotFound()) {
statuses[idx] = s;
assert(static_cast<size_t>(idx) < num_keys);
if (!status.ok() && !status.IsNotFound()) {
statuses[idx] = status;
} else {
value.assign(pinnable_val.data(), pinnable_val.size());
if (get_context.State() == GetContext::kFound) {
statuses[idx] = Status::OK();
}
@ -208,7 +195,6 @@ std::vector<Status> CompactedDBImpl::MultiGet(
}
++idx;
}
return statuses;
}
Status CompactedDBImpl::Init(const Options& options) {

21
db/db_impl/compacted_db_impl.h
View File

@ -27,26 +27,17 @@ class CompactedDBImpl : public DBImpl {
// Implementations of the DB interface
using DB::Get;
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) override;
Status Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value, std::string* timestamp) override;
const Slice& key, PinnableSlice* value,
std::string* timestamp) override;
using DB::MultiGet;
// Note that CompactedDBImpl::MultiGet is not the optimized version of
// MultiGet to use.
// TODO: optimize CompactedDBImpl::MultiGet, see DBImpl::MultiGet for details.
std::vector<Status> MultiGet(const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>&,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override;
std::vector<Status> MultiGet(const ReadOptions& _read_options,
const std::vector<ColumnFamilyHandle*>&,
const std::vector<Slice>& keys,
std::vector<std::string>* values,
std::vector<std::string>* timestamps) override;
void MultiGet(const ReadOptions& options, size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input) override;
using DBImpl::Put;
Status Put(const WriteOptions& /*options*/,

308
db/db_impl/db_impl.cc
View File

@ -2020,12 +2020,6 @@ ColumnFamilyHandle* DBImpl::PersistentStatsColumnFamily() const {
return persist_stats_cf_handle_;
}
Status DBImpl::Get(const ReadOptions& read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) {
return Get(read_options, column_family, key, value, /*timestamp=*/nullptr);
}
Status DBImpl::GetImpl(const ReadOptions& read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) {
@ -2497,250 +2491,6 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
return s;
}
std::vector<Status> DBImpl::MultiGet(
const ReadOptions& read_options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values) {
return MultiGet(read_options, column_family, keys, values,
/*timestamps=*/nullptr);
}
std::vector<Status> DBImpl::MultiGet(
const ReadOptions& _read_options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values,
std::vector<std::string>* timestamps) {
PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET);
PERF_TIMER_GUARD(get_snapshot_time);
size_t num_keys = keys.size();
assert(column_family.size() == num_keys);
std::vector<Status> stat_list(num_keys);
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kMultiGet) {
Status s = Status::InvalidArgument(
"Can only call MultiGet with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
for (size_t i = 0; i < num_keys; ++i) {
stat_list[i] = s;
}
return stat_list;
}
ReadOptions read_options(_read_options);
if (read_options.io_activity == Env::IOActivity::kUnknown) {
read_options.io_activity = Env::IOActivity::kMultiGet;
}
bool should_fail = false;
for (size_t i = 0; i < num_keys; ++i) {
assert(column_family[i]);
if (read_options.timestamp) {
stat_list[i] =
FailIfTsMismatchCf(column_family[i], *(read_options.timestamp));
if (!stat_list[i].ok()) {
should_fail = true;
}
} else {
stat_list[i] = FailIfCfHasTs(column_family[i]);
if (!stat_list[i].ok()) {
should_fail = true;
}
}
}
if (should_fail) {
for (auto& s : stat_list) {
if (s.ok()) {
s = Status::Incomplete(
"DB not queried due to invalid argument(s) in the same MultiGet");
}
}
return stat_list;
}
if (tracer_) {
// TODO: This mutex should be removed later, to improve performance when
// tracing is enabled.
InstrumentedMutexLock lock(&trace_mutex_);
if (tracer_) {
// TODO: maybe handle the tracing status?
tracer_->MultiGet(column_family, keys).PermitUncheckedError();
}
}
UnorderedMap<uint32_t, MultiGetColumnFamilyData> multiget_cf_data(
column_family.size());
for (auto cf : column_family) {
auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(cf);
auto cfd = cfh->cfd();
if (multiget_cf_data.find(cfd->GetID()) == multiget_cf_data.end()) {
multiget_cf_data.emplace(cfd->GetID(),
MultiGetColumnFamilyData(cfh, nullptr));
}
}
std::function<MultiGetColumnFamilyData*(
UnorderedMap<uint32_t, MultiGetColumnFamilyData>::iterator&)>
iter_deref_lambda =
[](UnorderedMap<uint32_t, MultiGetColumnFamilyData>::iterator&
cf_iter) { return &cf_iter->second; };
SequenceNumber consistent_seqnum;
bool sv_from_thread_local;
Status status =
MultiCFSnapshot<UnorderedMap<uint32_t, MultiGetColumnFamilyData>>(
read_options, nullptr, iter_deref_lambda, &multiget_cf_data,
&consistent_seqnum, &sv_from_thread_local);
if (!status.ok()) {
for (auto& s : stat_list) {
if (s.ok()) {
s = status;
}
}
return stat_list;
}
TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum1");
TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum2");
// Contain a list of merge operations if merge occurs.
MergeContext merge_context;
// Note: this always resizes the values array
values->resize(num_keys);
if (timestamps) {
timestamps->resize(num_keys);
}
// Keep track of bytes that we read for statistics-recording later
uint64_t bytes_read = 0;
PERF_TIMER_STOP(get_snapshot_time);
// For each of the given keys, apply the entire "get" process as follows:
// First look in the memtable, then in the immutable memtable (if any).
// s is both in/out. When in, s could either be OK or MergeInProgress.
// merge_operands will contain the sequence of merges in the latter case.
size_t num_found = 0;
size_t keys_read;
uint64_t curr_value_size = 0;
GetWithTimestampReadCallback timestamp_read_callback(0);
ReadCallback* read_callback = nullptr;
if (read_options.timestamp && read_options.timestamp->size() > 0) {
timestamp_read_callback.Refresh(consistent_seqnum);
read_callback = &timestamp_read_callback;
}
for (keys_read = 0; keys_read < num_keys; ++keys_read) {
merge_context.Clear();
Status& s = stat_list[keys_read];
std::string* value = &(*values)[keys_read];
std::string* timestamp = timestamps ? &(*timestamps)[keys_read] : nullptr;
LookupKey lkey(keys[keys_read], consistent_seqnum, read_options.timestamp);
auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
column_family[keys_read]);
SequenceNumber max_covering_tombstone_seq = 0;
auto mgd_iter = multiget_cf_data.find(cfh->cfd()->GetID());
assert(mgd_iter != multiget_cf_data.end());
auto mgd = mgd_iter->second;
auto super_version = mgd.super_version;
bool skip_memtable =
(read_options.read_tier == kPersistedTier &&
has_unpersisted_data_.load(std::memory_order_relaxed));
bool done = false;
if (!skip_memtable) {
if (super_version->mem->Get(
lkey, value, /*columns=*/nullptr, timestamp, &s, &merge_context,
&max_covering_tombstone_seq, read_options,
false /* immutable_memtable */, read_callback)) {
done = true;
RecordTick(stats_, MEMTABLE_HIT);
} else if (super_version->imm->Get(lkey, value, /*columns=*/nullptr,
timestamp, &s, &merge_context,
&max_covering_tombstone_seq,
read_options, read_callback)) {
done = true;
RecordTick(stats_, MEMTABLE_HIT);
}
}
if (!done) {
PinnableSlice pinnable_val;
PERF_TIMER_GUARD(get_from_output_files_time);
PinnedIteratorsManager pinned_iters_mgr;
super_version->current->Get(read_options, lkey, &pinnable_val,
/*columns=*/nullptr, timestamp, &s,
&merge_context, &max_covering_tombstone_seq,
&pinned_iters_mgr, /*value_found=*/nullptr,
/*key_exists=*/nullptr,
/*seq=*/nullptr, read_callback);
value->assign(pinnable_val.data(), pinnable_val.size());
RecordTick(stats_, MEMTABLE_MISS);
}
if (s.ok()) {
const auto& merge_threshold = read_options.merge_operand_count_threshold;
if (merge_threshold.has_value() &&
merge_context.GetNumOperands() > merge_threshold.value()) {
s = Status::OkMergeOperandThresholdExceeded();
}
bytes_read += value->size();
num_found++;
curr_value_size += value->size();
if (curr_value_size > read_options.value_size_soft_limit) {
while (++keys_read < num_keys) {
stat_list[keys_read] = Status::Aborted();
}
break;
}
}
if (read_options.deadline.count() &&
immutable_db_options_.clock->NowMicros() >
static_cast<uint64_t>(read_options.deadline.count())) {
break;
}
}
if (keys_read < num_keys) {
// The only reason to break out of the loop is when the deadline is
// exceeded
assert(immutable_db_options_.clock->NowMicros() >
static_cast<uint64_t>(read_options.deadline.count()));
for (++keys_read; keys_read < num_keys; ++keys_read) {
stat_list[keys_read] = Status::TimedOut();
}
}
// Post processing (decrement reference counts and record statistics)
PERF_TIMER_GUARD(get_post_process_time);
for (auto mgd_iter : multiget_cf_data) {
auto mgd = mgd_iter.second;
if (sv_from_thread_local) {
ReturnAndCleanupSuperVersion(mgd.cfd, mgd.super_version);
} else {
TEST_SYNC_POINT("DBImpl::MultiGet::BeforeLastTryUnRefSV");
CleanupSuperVersion(mgd.super_version);
}
}
RecordTick(stats_, NUMBER_MULTIGET_CALLS);
RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found);
RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read);
PERF_COUNTER_ADD(multiget_read_bytes, bytes_read);
PERF_TIMER_STOP(get_post_process_time);
return stat_list;
}
template <class T>
Status DBImpl::MultiCFSnapshot(
const ReadOptions& read_options, ReadCallback* callback,
@ -2889,7 +2639,8 @@ Status DBImpl::MultiCFSnapshot(
}
}
// Keep track of bytes that we read for statistics-recording later
TEST_SYNC_POINT("DBImpl::MultiCFSnapshot:AfterGetSeqNum1");
TEST_SYNC_POINT("DBImpl::MultiCFSnapshot:AfterGetSeqNum2");
PERF_TIMER_STOP(get_snapshot_time);
*sv_from_thread_local = !last_try;
if (!s.ok()) {
@ -2898,14 +2649,6 @@ Status DBImpl::MultiCFSnapshot(
return s;
}
void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool sorted_input) {
MultiGet(read_options, num_keys, column_families, keys, values,
/* timestamps */ nullptr, statuses, sorted_input);
}
void DBImpl::MultiGet(const ReadOptions& _read_options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
@ -3122,38 +2865,23 @@ void DBImpl::PrepareMultiGetKeys(
CompareKeyContext());
}
void DBImpl::MultiGet(const ReadOptions& read_options,
ColumnFamilyHandle* column_family, const size_t num_keys,
const Slice* keys, PinnableSlice* values,
Status* statuses, const bool sorted_input) {
MultiGet(read_options, column_family, num_keys, keys, values,
/* timestamps */ nullptr, statuses, sorted_input);
}
void DBImpl::MultiGet(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const size_t num_keys,
const Slice* keys, PinnableSlice* values,
std::string* timestamps, Status* statuses,
const bool sorted_input) {
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kMultiGet) {
Status s = Status::InvalidArgument(
"Can only call MultiGet with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
for (size_t i = 0; i < num_keys; ++i) {
if (statuses[i].ok()) {
statuses[i] = s;
}
}
return;
void DB::MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input) {
// Use std::array, if possible, to avoid memory allocation overhead
if (num_keys > MultiGetContext::MAX_BATCH_SIZE) {
std::vector<ColumnFamilyHandle*> column_families(num_keys, column_family);
MultiGet(options, num_keys, column_families.data(), keys, values,
timestamps, statuses, sorted_input);
} else {
std::array<ColumnFamilyHandle*, MultiGetContext::MAX_BATCH_SIZE>
column_families;
std::fill(column_families.begin(), column_families.begin() + num_keys,
column_family);
MultiGet(options, num_keys, column_families.data(), keys, values,
timestamps, statuses, sorted_input);
}
ReadOptions read_options(_read_options);
if (read_options.io_activity == Env::IOActivity::kUnknown) {
read_options.io_activity = Env::IOActivity::kMultiGet;
}
MultiGetCommon(read_options, column_family, num_keys, keys, values,
/* columns */ nullptr, timestamps, statuses, sorted_input);
}
void DBImpl::MultiGetCommon(const ReadOptions& read_options,

26
db/db_impl/db_impl.h
View File

@ -232,8 +232,6 @@ class DBImpl : public DB {
Status Write(const WriteOptions& options, WriteBatch* updates) override;
using DB::Get;
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) override;
Status Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value, std::string* timestamp) override;
@ -261,17 +259,6 @@ class DBImpl : public DB {
}
using DB::MultiGet;
std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override;
std::vector<Status> MultiGet(
const ReadOptions& _read_options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values,
std::vector<std::string>* timestamps) override;
// This MultiGet is a batched version, which may be faster than calling Get
// multiple times, especially if the keys have some spatial locality that
// enables them to be queried in the same SST files/set of files. The larger
@ -279,19 +266,6 @@ class DBImpl : public DB {
// The values and statuses parameters are arrays with number of elements
// equal to keys.size(). This allows the storage for those to be alloacted
// by the caller on the stack for small batches
void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
const size_t num_keys, const Slice* keys, PinnableSlice* values,
Status* statuses, const bool sorted_input = false) override;
void MultiGet(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const size_t num_keys,
const Slice* keys, PinnableSlice* values,
std::string* timestamps, Status* statuses,
const bool sorted_input = false) override;
void MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool sorted_input = false) override;
void MultiGet(const ReadOptions& _read_options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,

18
db/db_test.cc
View File

@ -3098,7 +3098,8 @@ class ModelDB : public DB {
}
using DB::Get;
Status Get(const ReadOptions& /*options*/, ColumnFamilyHandle* /*cf*/,
const Slice& key, PinnableSlice* /*value*/) override {
const Slice& key, PinnableSlice* /*value*/,
std::string* /*timestamp*/) override {
return Status::NotSupported(key);
}
@ -3112,14 +3113,13 @@ class ModelDB : public DB {
}
using DB::MultiGet;
std::vector<Status> MultiGet(
const ReadOptions& /*options*/,
const std::vector<ColumnFamilyHandle*>& /*column_family*/,
const std::vector<Slice>& keys,
std::vector<std::string>* /*values*/) override {
std::vector<Status> s(keys.size(),
Status::NotSupported("Not implemented."));
return s;
void MultiGet(const ReadOptions& /*options*/, const size_t num_keys,
ColumnFamilyHandle** /*column_families*/, const Slice* /*keys*/,
PinnableSlice* /*values*/, std::string* /*timestamps*/,
Status* statuses, const bool /*sorted_input*/) override {
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] = Status::NotSupported("Not implemented.");
}
}
using DB::IngestExternalFile;

6
db/db_with_timestamp_basic_test.cc
View File

@ -2610,10 +2610,10 @@ TEST_F(DataVisibilityTest, MultiGetWithoutSnapshot) {
SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->LoadDependency({
{"DBImpl::MultiGet:AfterGetSeqNum1",
{"DBImpl::MultiCFSnapshot:AfterGetSeqNum1",
"DataVisibilityTest::MultiGetWithoutSnapshot:BeforePut"},
{"DataVisibilityTest::MultiGetWithoutSnapshot:AfterPut",
"DBImpl::MultiGet:AfterGetSeqNum2"},
"DBImpl::MultiCFSnapshot:AfterGetSeqNum2"},
});
SyncPoint::GetInstance()->EnableProcessing();
port::Thread writer_thread([this]() {
@ -4737,4 +4737,4 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
RegisterCustomObjects(argc, argv);
return RUN_ALL_TESTS();
}
}

267
include/rocksdb/db.h
View File

@ -549,35 +549,23 @@ class DB {
// any, or an empty value otherwise.
//
// If timestamp is enabled and a non-null timestamp pointer is passed in,
// timestamp is returned.
// timestamp is returned. If the underlying DB implementation doesn't
// support returning timestamp and the timestamp argument is non-null,
// a Status::NotSupported() error will be returned.
//
// Returns OK on success. Returns NotFound and an empty value in "*value" if
// there is no entry for "key". Returns some other non-OK status on error.
virtual inline Status Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
std::string* value) {
assert(value != nullptr);
PinnableSlice pinnable_val(value);
assert(!pinnable_val.IsPinned());
auto s = Get(options, column_family, key, &pinnable_val);
if (s.ok() && pinnable_val.IsPinned()) {
value->assign(pinnable_val.data(), pinnable_val.size());
} // else value is already assigned
return s;
}
// NOTE: Pure virtual => was virtual before
virtual Status Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) = 0;
virtual Status Get(const ReadOptions& options, const Slice& key,
std::string* value) {
return Get(options, DefaultColumnFamily(), key, value);
}
PinnableSlice* value, std::string* timestamp) = 0;
// Get() methods that return timestamp. Derived DB classes don't need to worry
// about this group of methods if they don't care about timestamp feature.
// The timestamp of the key is returned if a non-null timestamp pointer is
// passed, and value is returned as a string
// NOTE: virtual final => disallow override (was previously allowed)
virtual inline Status Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
std::string* value, std::string* timestamp) {
std::string* value, std::string* timestamp) final {
assert(value != nullptr);
PinnableSlice pinnable_val(value);
assert(!pinnable_val.IsPinned());
@ -587,15 +575,43 @@ class DB {
} // else value is already assigned
return s;
}
virtual Status Get(const ReadOptions& /*options*/,
ColumnFamilyHandle* /*column_family*/,
const Slice& /*key*/, PinnableSlice* /*value*/,
std::string* /*timestamp*/) {
return Status::NotSupported(
"Get() that returns timestamp is not implemented.");
// No timestamp, and value is returned in a PinnableSlice
// NOTE: virtual final => disallow override (was previously allowed)
virtual Status Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) final {
return Get(options, column_family, key, value, nullptr);
}
// No timestamp, and the value is returned as a string
// NOTE: virtual final => disallow override (was previously allowed)
virtual inline Status Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
std::string* value) final {
assert(value != nullptr);
PinnableSlice pinnable_val(value);
assert(!pinnable_val.IsPinned());
auto s = Get(options, column_family, key, &pinnable_val);
if (s.ok() && pinnable_val.IsPinned()) {
value->assign(pinnable_val.data(), pinnable_val.size());
} // else value is already assigned
return s;
}
// Gets a key in the default column family, returns the value as a string,
// and no timestamp returned
// NOTE: virtual final => disallow override (was previously allowed)
virtual Status Get(const ReadOptions& options, const Slice& key,
std::string* value, std::string* timestamp) {
std::string* value) final {
return Get(options, DefaultColumnFamily(), key, value);
}
// Gets a key in the default column family, returns the value as a string,
// and timestamp of the key is returned if timestamp parameter is non-null
// NOTE: virtual final => disallow override (was previously allowed)
virtual Status Get(const ReadOptions& options, const Slice& key,
std::string* value, std::string* timestamp) final {
return Get(options, DefaultColumnFamily(), key, value, timestamp);
}
@ -650,9 +666,10 @@ class DB {
int* number_of_operands) = 0;
// Consistent Get of many keys across column families without the need
// for an explicit snapshot. NOTE: the implementation of this MultiGet API
// does not have the performance benefits of the void-returning MultiGet
// functions.
// for an explicit snapshot. The main difference between this set of
// MultiGet APis and the batched MultiGet APIs that follow are -
// 1. The APIs take std::vector instead of C style array pointers
// 2. Values are returned as std::string rather than PinnableSlice
//
// If keys[i] does not exist in the database, then the i'th returned
// status will be one for which Status::IsNotFound() is true, and
@ -662,35 +679,67 @@ class DB {
//
// (*values) will always be resized to be the same size as (keys).
// Similarly, the number of returned statuses will be the number of keys.
// If timestamps is non-null, the vector pointed to by it will be resized to
// number of keys and filled with timestamps of the keys on return.
// Note: keys will not be "de-duplicated". Duplicate keys will return
// duplicate values in order, and may return different status values
// in case there are errors.
// NOTE: virtual final => disallow override (was previously allowed)
virtual std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_families,
const std::vector<Slice>& keys, std::vector<std::string>* values,
std::vector<std::string>* timestamps) final {
size_t num_keys = keys.size();
std::vector<Status> statuses(num_keys);
std::vector<PinnableSlice> pin_values(num_keys);
values->resize(num_keys);
if (timestamps) {
timestamps->resize(num_keys);
}
MultiGet(options, num_keys,
const_cast<ColumnFamilyHandle**>(column_families.data()),
keys.data(), pin_values.data(),
timestamps ? timestamps->data() : nullptr, statuses.data(),
/*sorted_input=*/false);
for (size_t i = 0; i < num_keys; ++i) {
if (statuses[i].ok()) {
(*values)[i].assign(pin_values[i].data(), pin_values[i].size());
}
}
return statuses;
}
// No timestamps are returned
// NOTE: virtual final => disallow override (was previously allowed)
virtual std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values) = 0;
const std::vector<Slice>& keys, std::vector<std::string>* values) final {
values->resize(keys.size());
return MultiGet(options, column_family, keys, values, nullptr);
}
// MultiGet for default column family, no timestamps returned
// NOTE: virtual final => disallow override (was previously allowed)
virtual std::vector<Status> MultiGet(const ReadOptions& options,
const std::vector<Slice>& keys,
std::vector<std::string>* values) {
std::vector<std::string>* values) final {
values->resize(keys.size());
return MultiGet(
options,
std::vector<ColumnFamilyHandle*>(keys.size(), DefaultColumnFamily()),
keys, values);
}
// MultiGet for default column family
// NOTE: virtual final => disallow override (was previously allowed)
virtual std::vector<Status> MultiGet(
const ReadOptions& /*options*/,
const std::vector<ColumnFamilyHandle*>& /*column_family*/,
const std::vector<Slice>& keys, std::vector<std::string>* /*values*/,
std::vector<std::string>* /*timestamps*/) {
return std::vector<Status>(
keys.size(), Status::NotSupported(
"MultiGet() returning timestamps not implemented."));
}
virtual std::vector<Status> MultiGet(const ReadOptions& options,
const std::vector<Slice>& keys,
std::vector<std::string>* values,
std::vector<std::string>* timestamps) {
const ReadOptions& options, const std::vector<Slice>& keys,
std::vector<std::string>* values,
std::vector<std::string>* timestamps) final {
values->resize(keys.size());
return MultiGet(
options,
std::vector<ColumnFamilyHandle*>(keys.size(), DefaultColumnFamily()),
@ -705,123 +754,59 @@ class DB {
// benefits.
// Parameters -
// options - ReadOptions
// column_family - ColumnFamilyHandle* that the keys belong to. All the keys
// passed to the API are restricted to a single column family
// num_keys - Number of keys to lookup
// column_families - Pointer to C style array of ColumnFamilyHandle* that
// the keys belong to.
// keys - Pointer to C style array of key Slices with num_keys elements
// values - Pointer to C style array of PinnableSlices with num_keys elements
// timestamps - Pointer to C style array of std::string that, if non-null and
// timestamps are enabled, will be filled with timestamps of the
// keys on return. The array should be sized to num_keys entries
// by the caller.
// statuses - Pointer to C style array of Status with num_keys elements
// sorted_input - If true, it means the input keys are already sorted by key
// order, so the MultiGet() API doesn't have to sort them
// again. If false, the keys will be copied and sorted
// internally by the API - the input array will not be
// modified
virtual void MultiGet(const ReadOptions& options,
ColumnFamilyHandle* column_family,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool /*sorted_input*/ = false) {
std::vector<ColumnFamilyHandle*> cf;
std::vector<Slice> user_keys;
std::vector<Status> status;
std::vector<std::string> vals;
for (size_t i = 0; i < num_keys; ++i) {
cf.emplace_back(column_family);
user_keys.emplace_back(keys[i]);
}
status = MultiGet(options, cf, user_keys, &vals);
std::copy(status.begin(), status.end(), statuses);
for (auto& value : vals) {
values->PinSelf(value);
values++;
}
}
// NOTE: Pure virtual => was virtual (optional). If the concrete
// implementation
// doesn't support returning timestamps, and the timestamps paramater is
// non-null, it should return Status::NotSupported() for all the keys.
virtual void MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input = false) = 0;
// MultiGet for single column family
// NOTE: virtual final => disallow override (was previously allowed)
virtual void MultiGet(const ReadOptions& options,
ColumnFamilyHandle* column_family,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool /*sorted_input*/ = false) {
std::vector<ColumnFamilyHandle*> cf;
std::vector<Slice> user_keys;
std::vector<Status> status;
std::vector<std::string> vals;
std::vector<std::string> tss;
Status* statuses,
const bool sorted_input = false) final;
for (size_t i = 0; i < num_keys; ++i) {
cf.emplace_back(column_family);
user_keys.emplace_back(keys[i]);
}
status = MultiGet(options, cf, user_keys, &vals, &tss);
std::copy(status.begin(), status.end(), statuses);
std::copy(tss.begin(), tss.end(), timestamps);
for (auto& value : vals) {
values->PinSelf(value);
values++;
}
// MultiGet for single column family, no timestamps returned
// NOTE: virtual final => disallow override (was previously allowed)
virtual void MultiGet(const ReadOptions& options,
ColumnFamilyHandle* column_family,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool sorted_input = false) final {
MultiGet(options, column_family, num_keys, keys, values, nullptr, statuses,
sorted_input);
}
// Overloaded MultiGet API that improves performance by batching operations
// in the read path for greater efficiency. Currently, only the block based
// table format with full filters are supported. Other table formats such
// as plain table, block based table with block based filters and
// partitioned indexes will still work, but will not get any performance
// benefits.
// Parameters -
// options - ReadOptions
// column_family - ColumnFamilyHandle* that the keys belong to. All the keys
// passed to the API are restricted to a single column family
// num_keys - Number of keys to lookup
// keys - Pointer to C style array of key Slices with num_keys elements
// values - Pointer to C style array of PinnableSlices with num_keys elements
// statuses - Pointer to C style array of Status with num_keys elements
// sorted_input - If true, it means the input keys are already sorted by key
// order, so the MultiGet() API doesn't have to sort them
// again. If false, the keys will be copied and sorted
// internally by the API - the input array will not be
// modified
// Multiple column families, no timestamps returned
// NOTE: virtual final => disallow override (was previously allowed)
virtual void MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool /*sorted_input*/ = false) {
std::vector<ColumnFamilyHandle*> cf;
std::vector<Slice> user_keys;
std::vector<Status> status;
std::vector<std::string> vals;
for (size_t i = 0; i < num_keys; ++i) {
cf.emplace_back(column_families[i]);
user_keys.emplace_back(keys[i]);
}
status = MultiGet(options, cf, user_keys, &vals);
std::copy(status.begin(), status.end(), statuses);
for (auto& value : vals) {
values->PinSelf(value);
values++;
}
}
virtual void MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool /*sorted_input*/ = false) {
std::vector<ColumnFamilyHandle*> cf;
std::vector<Slice> user_keys;
std::vector<Status> status;
std::vector<std::string> vals;
std::vector<std::string> tss;
for (size_t i = 0; i < num_keys; ++i) {
cf.emplace_back(column_families[i]);
user_keys.emplace_back(keys[i]);
}
status = MultiGet(options, cf, user_keys, &vals, &tss);
std::copy(status.begin(), status.end(), statuses);
std::copy(tss.begin(), tss.end(), timestamps);
for (auto& value : vals) {
values->PinSelf(value);
values++;
}
const bool sorted_input = false) final {
MultiGet(options, num_keys, column_families, keys, values, nullptr,
statuses, sorted_input);
}
// Batched MultiGet-like API that returns wide-column entities from a single

22
include/rocksdb/utilities/stackable_db.h
View File

@ -97,8 +97,9 @@ class StackableDB : public DB {
using DB::Get;
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) override {
return db_->Get(options, column_family, key, value);
const Slice& key, PinnableSlice* value,
std::string* timestamp) override {
return db_->Get(options, column_family, key, value, timestamp);
}
using DB::GetEntity;
@ -120,19 +121,12 @@ class StackableDB : public DB {
}
using DB::MultiGet;
std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override {
return db_->MultiGet(options, column_family, keys, values);
}
void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
const size_t num_keys, const Slice* keys, PinnableSlice* values,
void MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input = false) override {
return db_->MultiGet(options, column_family, num_keys, keys, values,
statuses, sorted_input);
return db_->MultiGet(options, num_keys, column_families, keys, values,
timestamps, statuses, sorted_input);
}
using DB::MultiGetEntity;

5
java/src/test/java/org/rocksdb/VerifyChecksumsTest.java
View File

@ -166,9 +166,10 @@ public class VerifyChecksumsTest {
statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT);
if (verifyFlag) {
// We don't need to be exact - we are checking that the checksums happen
// exactly how many depends on block size etc etc, so may not be entirely stable
// exactly how many depends on block size, MultiGet batching etc etc,
// so may not be entirely stable
System.out.println(MessageFormat.format("verify=true {0}", afterOperationsCount));
assertThat(afterOperationsCount).isGreaterThan(beforeOperationsCount + 20);
assertThat(afterOperationsCount).isGreaterThan(beforeOperationsCount);
} else {
System.out.println(MessageFormat.format("verify=false {0}", afterOperationsCount));
assertThat(afterOperationsCount).isEqualTo(beforeOperationsCount);

1
table/block_based/block_based_table_reader_sync_and_async.h
View File

@ -222,6 +222,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
// beyond the payload size.
s = VerifyBlockChecksum(footer, data + req_offset, handle.size(),
rep_->file->file_name(), handle.offset());
RecordTick(ioptions.stats, BLOCK_CHECKSUM_COMPUTE_COUNT);
TEST_SYNC_POINT_CALLBACK("RetrieveMultipleBlocks:VerifyChecksum", &s);
}
} else if (!use_shared_buffer) {

1
unreleased_history/public_api_changes/consolidate_get_mget.md Normal file
View File

@ -0,0 +1 @@
Declare one Get and one MultiGet variant as pure virtual, and make all the other variants non-overridable. The methods required to be implemented by derived classes of DB allow returning timestamps. It is up to the implementation to check and return an error if timestamps are not supported. The non-batched MultiGet APIs are reimplemented in terms of batched MultiGet, so callers might see a performance improvement.

33
utilities/blob_db/blob_db.h
View File

@ -138,7 +138,8 @@ class BlobDB : public StackableDB {
using ROCKSDB_NAMESPACE::StackableDB::Get;
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) override = 0;
const Slice& key, PinnableSlice* value,
std::string* timestamp) override = 0;
// Get value and expiration.
virtual Status Get(const ReadOptions& options,
@ -149,36 +150,6 @@ class BlobDB : public StackableDB {
return Get(options, DefaultColumnFamily(), key, value, expiration);
}
using ROCKSDB_NAMESPACE::StackableDB::MultiGet;
std::vector<Status> MultiGet(const ReadOptions& options,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override = 0;
std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_families,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override {
for (auto column_family : column_families) {
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
return std::vector<Status>(
column_families.size(),
Status::NotSupported(
"Blob DB doesn't support non-default column family."));
}
}
return MultiGet(options, keys, values);
}
void MultiGet(const ReadOptions& /*options*/,
ColumnFamilyHandle* /*column_family*/, const size_t num_keys,
const Slice* /*keys*/, PinnableSlice* /*values*/,
Status* statuses,
const bool /*sorted_input*/ = false) override {
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] =
Status::NotSupported("Blob DB doesn't support batched MultiGet");
}
}
using ROCKSDB_NAMESPACE::StackableDB::SingleDelete;
Status SingleDelete(const WriteOptions& /*wopts*/,
ColumnFamilyHandle* /*column_family*/,

66
utilities/blob_db/blob_db_impl.cc
View File

@ -1409,27 +1409,43 @@ Status BlobDBImpl::AppendBlob(const WriteOptions& write_options,
return s;
}
std::vector<Status> BlobDBImpl::MultiGet(const ReadOptions& _read_options,
const std::vector<Slice>& keys,
std::vector<std::string>* values) {
void BlobDBImpl::MultiGet(const ReadOptions& _read_options, size_t num_keys,
ColumnFamilyHandle** column_families,
const Slice* keys, PinnableSlice* values,
std::string* timestamps, Status* statuses,
const bool /*sorted_input*/) {
StopWatch multiget_sw(clock_, statistics_, BLOB_DB_MULTIGET_MICROS);
RecordTick(statistics_, BLOB_DB_NUM_MULTIGET);
// Get a snapshot to avoid blob file get deleted between we
// fetch and index entry and reading from the file.
std::vector<Status> statuses;
std::size_t num_keys = keys.size();
statuses.reserve(num_keys);
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kMultiGet) {
Status s = Status::InvalidArgument(
"Can only call MultiGet with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
for (size_t i = 0; i < num_keys; ++i) {
statuses.push_back(s);
{
Status s;
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kMultiGet) {
s = Status::InvalidArgument(
"Can only call MultiGet with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
} else if (timestamps) {
s = Status::NotSupported(
"MultiGet() returning timestamps not implemented.");
}
if (s.ok()) {
for (size_t i = 0; i < num_keys; ++i) {
if (column_families[i]->GetID() != DefaultColumnFamily()->GetID()) {
s = Status::NotSupported(
"Blob DB doesn't support non-default column family.");
break;
}
}
}
if (!s.ok()) {
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] = s;
}
return;
}
return statuses;
}
ReadOptions read_options(_read_options);
@ -1438,19 +1454,14 @@ std::vector<Status> BlobDBImpl::MultiGet(const ReadOptions& _read_options,
}
bool snapshot_created = SetSnapshotIfNeeded(&read_options);
values->clear();
values->reserve(keys.size());
PinnableSlice value;
for (size_t i = 0; i < keys.size(); i++) {
statuses.push_back(
GetImpl(read_options, DefaultColumnFamily(), keys[i], &value));
values->push_back(value.ToString());
value.Reset();
for (size_t i = 0; i < num_keys; i++) {
PinnableSlice& value = values[i];
statuses[i] = GetImpl(read_options, DefaultColumnFamily(), keys[i], &value);
}
if (snapshot_created) {
db_->ReleaseSnapshot(read_options.snapshot);
}
return statuses;
return;
}
bool BlobDBImpl::SetSnapshotIfNeeded(ReadOptions* read_options) {
@ -1654,13 +1665,18 @@ Status BlobDBImpl::GetRawBlobFromFile(const Slice& key, uint64_t file_number,
Status BlobDBImpl::Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) {
PinnableSlice* value, std::string* timestamp) {
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kGet) {
return Status::InvalidArgument(
"Can only call Get with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`");
}
if (timestamp) {
return Status::NotSupported(
"Get() that returns timestamp is not implemented.");
}
ReadOptions read_options(_read_options);
if (read_options.io_activity == Env::IOActivity::kUnknown) {
read_options.io_activity = Env::IOActivity::kGet;

9
utilities/blob_db/blob_db_impl.h
View File

@ -105,7 +105,7 @@ class BlobDBImpl : public BlobDB {
using BlobDB::Get;
Status Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) override;
PinnableSlice* value, std::string* timestamp) override;
Status Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const Slice& key,
@ -123,9 +123,10 @@ class BlobDBImpl : public BlobDB {
}
using BlobDB::MultiGet;
std::vector<Status> MultiGet(const ReadOptions& _read_options,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override;
void MultiGet(const ReadOptions& _read_options, size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input) override;
using BlobDB::Write;
Status Write(const WriteOptions& opts, WriteBatch* updates) override;

45
utilities/transactions/write_prepared_txn_db.cc
View File

@ -249,13 +249,18 @@ Status WritePreparedTxnDB::WriteInternal(const WriteOptions& write_options_orig,
Status WritePreparedTxnDB::Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) {
const Slice& key, PinnableSlice* value,
std::string* timestamp) {
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kGet) {
return Status::InvalidArgument(
"Can only call Get with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`");
}
if (timestamp) {
return Status::NotSupported(
"Get() that returns timestamp is not implemented");
}
ReadOptions read_options(_read_options);
if (read_options.io_activity == Env::IOActivity::kUnknown) {
read_options.io_activity = Env::IOActivity::kGet;
@ -325,24 +330,34 @@ void WritePreparedTxnDB::UpdateCFComparatorMap(ColumnFamilyHandle* h) {
handle_map_.reset(handle_map);
}
std::vector<Status> WritePreparedTxnDB::MultiGet(
const ReadOptions& _read_options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values) {
void WritePreparedTxnDB::MultiGet(const ReadOptions& _read_options,
const size_t num_keys,
ColumnFamilyHandle** column_families,
const Slice* keys, PinnableSlice* values,
std::string* timestamps, Status* statuses,
const bool /*sorted_input*/) {
assert(values);
size_t num_keys = keys.size();
std::vector<Status> stat_list(num_keys);
Status s;
if (_read_options.io_activity != Env::IOActivity::kUnknown &&
_read_options.io_activity != Env::IOActivity::kMultiGet) {
Status s = Status::InvalidArgument(
s = Status::InvalidArgument(
"Can only call MultiGet with `ReadOptions::io_activity` is "
"`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
}
for (size_t i = 0; i < num_keys; ++i) {
stat_list[i] = s;
if (s.ok()) {
if (timestamps) {
s = Status::NotSupported(
"MultiGet() returning timestamps not implemented.");
}
return stat_list;
}
if (!s.ok()) {
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] = s;
}
return;
}
ReadOptions read_options(_read_options);
@ -350,13 +365,11 @@ std::vector<Status> WritePreparedTxnDB::MultiGet(
read_options.io_activity = Env::IOActivity::kMultiGet;
}
values->resize(num_keys);
for (size_t i = 0; i < num_keys; ++i) {
stat_list[i] =
this->GetImpl(read_options, column_family[i], keys[i], &(*values)[i]);
statuses[i] =
this->GetImpl(read_options, column_families[i], keys[i], &values[i]);
}
return stat_list;
return;
}
// Struct to hold ownership of snapshot and read callback for iterator cleanup.

11
utilities/transactions/write_prepared_txn_db.h
View File

@ -84,14 +84,13 @@ class WritePreparedTxnDB : public PessimisticTransactionDB {
using DB::Get;
Status Get(const ReadOptions& _read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) override;
PinnableSlice* value, std::string* timestamp) override;
using DB::MultiGet;
std::vector<Status> MultiGet(
const ReadOptions& _read_options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override;
void MultiGet(const ReadOptions& _read_options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input) override;
using DB::NewIterator;
Iterator* NewIterator(const ReadOptions& _read_options,

46
utilities/ttl/db_ttl_impl.cc
View File

@ -493,7 +493,11 @@ Status DBWithTTLImpl::Put(const WriteOptions& options,
Status DBWithTTLImpl::Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) {
PinnableSlice* value, std::string* timestamp) {
if (timestamp) {
return Status::NotSupported(
"Get() that returns timestamp is not supported");
}
Status st = db_->Get(options, column_family, key, value);
if (!st.ok()) {
return st;
@ -505,22 +509,34 @@ Status DBWithTTLImpl::Get(const ReadOptions& options,
return StripTS(value);
}
std::vector<Status> DBWithTTLImpl::MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values) {
auto statuses = db_->MultiGet(options, column_family, keys, values);
for (size_t i = 0; i < keys.size(); ++i) {
if (!statuses[i].ok()) {
continue;
void DBWithTTLImpl::MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families,
const Slice* keys, PinnableSlice* values,
std::string* timestamps, Status* statuses,
const bool /*sorted_input*/) {
if (timestamps) {
for (size_t i = 0; i < num_keys; ++i) {
statuses[i] = Status::NotSupported(
"MultiGet() returning timestamps not implemented.");
}
statuses[i] = SanityCheckTimestamp((*values)[i]);
if (!statuses[i].ok()) {
continue;
}
statuses[i] = StripTS(&(*values)[i]);
return;
}
db_->MultiGet(options, num_keys, column_families, keys, values, timestamps,
statuses);
for (size_t i = 0; i < num_keys; ++i) {
if (!statuses[i].ok()) {
continue;
}
PinnableSlice tmp_val = std::move(values[i]);
values[i].PinSelf(tmp_val);
assert(!values[i].IsPinned());
statuses[i] = SanityCheckTimestamp(values[i]);
if (!statuses[i].ok()) {
continue;
}
statuses[i] = StripTS(&values[i]);
}
return statuses;
}
bool DBWithTTLImpl::KeyMayExist(const ReadOptions& options,

12
utilities/ttl/db_ttl_impl.h
View File

@ -53,14 +53,14 @@ class DBWithTTLImpl : public DBWithTTL {
using StackableDB::Get;
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) override;
const Slice& key, PinnableSlice* value,
std::string* timestamp) override;
using StackableDB::MultiGet;
std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys,
std::vector<std::string>* values) override;
void MultiGet(const ReadOptions& options, const size_t num_keys,
ColumnFamilyHandle** column_families, const Slice* keys,
PinnableSlice* values, std::string* timestamps,
Status* statuses, const bool sorted_input) override;
using StackableDB::KeyMayExist;
bool KeyMayExist(const ReadOptions& options,