2019-12-16 22:28:06 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#ifdef GFLAGS
|
|
|
|
#pragma once
|
|
|
|
#include "db_stress_tool/db_stress_common.h"
|
2023-04-21 16:07:18 +00:00
|
|
|
#include "monitoring/thread_status_util.h"
|
2019-12-16 22:28:06 +00:00
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2023-04-21 16:07:18 +00:00
|
|
|
class DbStressRandomAccessFileWrapper : public FSRandomAccessFileOwnerWrapper {
|
|
|
|
public:
|
|
|
|
explicit DbStressRandomAccessFileWrapper(
|
|
|
|
std::unique_ptr<FSRandomAccessFile>&& target)
|
|
|
|
: FSRandomAccessFileOwnerWrapper(std::move(target)) {}
|
|
|
|
|
|
|
|
IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
|
|
|
|
Slice* result, char* scratch,
|
|
|
|
IODebugContext* dbg) const override {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
const ThreadStatus::OperationType thread_op =
|
|
|
|
ThreadStatusUtil::GetThreadOperation();
|
|
|
|
Env::IOActivity io_activity =
|
|
|
|
ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op);
|
|
|
|
assert(io_activity == Env::IOActivity::kUnknown ||
|
|
|
|
io_activity == options.io_activity);
|
|
|
|
#endif
|
|
|
|
return target()->Read(offset, n, options, result, scratch, dbg);
|
|
|
|
}
|
Group rocksdb.sst.read.micros stat by different user read IOActivity + misc (#11444)
Summary:
**Context/Summary:**
- Similar to https://github.com/facebook/rocksdb/pull/11288 but for user read such as `Get(), MultiGet(), DBIterator::XXX(), Verify(File)Checksum()`.
- For this, I refactored some user-facing `MultiGet` calls in `TransactionBase` and various types of `DB` so that it does not call a user-facing `Get()` but `GetImpl()` for passing the `ReadOptions::io_activity` check (see PR conversation)
- New user read stats breakdown are guarded by `kExceptDetailedTimers` since measurement shows they have 4-5% regression to the upstream/main.
- Misc
- More refactoring: with https://github.com/facebook/rocksdb/pull/11288, we complete passing `ReadOptions/IOOptions` to FS level. So we can now replace the previously [added](https://github.com/facebook/rocksdb/pull/9424) `rate_limiter_priority` parameter in `RandomAccessFileReader`'s `Read/MultiRead/Prefetch()` with `IOOptions::rate_limiter_priority`
- Also, `ReadAsync()` call time is measured in `SST_READ_MICRO` now
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11444
Test Plan:
- CI fake db crash/stress test
- Microbenchmarking
**Build** `make clean && ROCKSDB_NO_FBCODE=1 DEBUG_LEVEL=0 make -jN db_basic_bench`
- google benchmark version: https://github.com/google/benchmark/commit/604f6fd3f4b34a84ec4eb4db81d842fa4db829cd
- db_basic_bench_base: upstream
- db_basic_bench_pr: db_basic_bench_base + this PR
- asyncread_db_basic_bench_base: upstream + [db basic bench patch for IteratorNext](https://github.com/facebook/rocksdb/compare/main...hx235:rocksdb:micro_bench_async_read)
- asyncread_db_basic_bench_pr: asyncread_db_basic_bench_base + this PR
**Test**
Get
```
TEST_TMPDIR=/dev/shm ./db_basic_bench_{null_stat|base|pr} --benchmark_filter=DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/mmap:1/threads:1 --benchmark_repetitions=1000
```
Result
```
Coming soon
```
AsyncRead
```
TEST_TMPDIR=/dev/shm ./asyncread_db_basic_bench_{base|pr} --benchmark_filter=IteratorNext/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/async_io:1/include_detailed_timers:0 --benchmark_repetitions=1000 > syncread_db_basic_bench_{base|pr}.out
```
Result
```
Base:
1956,1956,1968,1977,1979,1986,1988,1988,1988,1990,1991,1991,1993,1993,1993,1993,1994,1996,1997,1997,1997,1998,1999,2001,2001,2002,2004,2007,2007,2008,
PR (2.3% regression, due to measuring `SST_READ_MICRO` that wasn't measured before):
1993,2014,2016,2022,2024,2027,2027,2028,2028,2030,2031,2031,2032,2032,2038,2039,2042,2044,2044,2047,2047,2047,2048,2049,2050,2052,2052,2052,2053,2053,
```
Reviewed By: ajkr
Differential Revision: D45918925
Pulled By: hx235
fbshipit-source-id: 58a54560d9ebeb3a59b6d807639692614dad058a
2023-08-09 00:26:50 +00:00
|
|
|
|
|
|
|
IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs,
|
|
|
|
const IOOptions& options, IODebugContext* dbg) override {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
const ThreadStatus::OperationType thread_op =
|
|
|
|
ThreadStatusUtil::GetThreadOperation();
|
|
|
|
Env::IOActivity io_activity =
|
|
|
|
ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op);
|
|
|
|
assert(io_activity == Env::IOActivity::kUnknown ||
|
|
|
|
io_activity == options.io_activity);
|
|
|
|
#endif
|
|
|
|
return target()->MultiRead(reqs, num_reqs, options, dbg);
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options,
|
|
|
|
IODebugContext* dbg) override {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
const ThreadStatus::OperationType thread_op =
|
|
|
|
ThreadStatusUtil::GetThreadOperation();
|
|
|
|
Env::IOActivity io_activity =
|
|
|
|
ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op);
|
|
|
|
assert(io_activity == Env::IOActivity::kUnknown ||
|
|
|
|
io_activity == options.io_activity);
|
|
|
|
#endif
|
|
|
|
return target()->Prefetch(offset, n, options, dbg);
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus ReadAsync(FSReadRequest& req, const IOOptions& options,
|
|
|
|
std::function<void(const FSReadRequest&, void*)> cb,
|
|
|
|
void* cb_arg, void** io_handle, IOHandleDeleter* del_fn,
|
|
|
|
IODebugContext* dbg) override {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
const ThreadStatus::OperationType thread_op =
|
|
|
|
ThreadStatusUtil::GetThreadOperation();
|
|
|
|
Env::IOActivity io_activity =
|
|
|
|
ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op);
|
|
|
|
assert(io_activity == Env::IOActivity::kUnknown ||
|
|
|
|
io_activity == options.io_activity);
|
|
|
|
#endif
|
|
|
|
return target()->ReadAsync(req, options, cb, cb_arg, io_handle, del_fn,
|
|
|
|
dbg);
|
|
|
|
}
|
2023-04-21 16:07:18 +00:00
|
|
|
};
|
|
|
|
|
2022-12-15 23:48:50 +00:00
|
|
|
class DbStressFSWrapper : public FileSystemWrapper {
|
2019-12-16 22:28:06 +00:00
|
|
|
public:
|
2022-12-15 23:48:50 +00:00
|
|
|
explicit DbStressFSWrapper(const std::shared_ptr<FileSystem>& t)
|
|
|
|
: FileSystemWrapper(t) {}
|
|
|
|
static const char* kClassName() { return "DbStressFS"; }
|
2022-01-05 00:44:54 +00:00
|
|
|
const char* Name() const override { return kClassName(); }
|
2019-12-16 22:28:06 +00:00
|
|
|
|
2023-04-21 16:07:18 +00:00
|
|
|
IOStatus NewRandomAccessFile(const std::string& f,
|
|
|
|
const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSRandomAccessFile>* r,
|
|
|
|
IODebugContext* dbg) override {
|
|
|
|
std::unique_ptr<FSRandomAccessFile> file;
|
|
|
|
IOStatus s = target()->NewRandomAccessFile(f, file_opts, &file, dbg);
|
|
|
|
if (s.ok()) {
|
|
|
|
r->reset(new DbStressRandomAccessFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2022-12-15 23:48:50 +00:00
|
|
|
IOStatus DeleteFile(const std::string& f, const IOOptions& opts,
|
|
|
|
IODebugContext* dbg) override {
|
2019-12-16 22:28:06 +00:00
|
|
|
// We determine whether it is a manifest file by searching a strong,
|
|
|
|
// so that there will be false positive if the directory path contains the
|
|
|
|
// keyword but it is unlikely.
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
2020-09-04 03:11:45 +00:00
|
|
|
// Checkpoint, backup, and restore directories needs to be exempted.
|
2019-12-16 22:28:06 +00:00
|
|
|
if (!if_preserve_all_manifests ||
|
2019-12-21 06:08:44 +00:00
|
|
|
f.find("MANIFEST-") == std::string::npos ||
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
2020-09-04 03:11:45 +00:00
|
|
|
f.find("checkpoint") != std::string::npos ||
|
|
|
|
f.find(".backup") != std::string::npos ||
|
|
|
|
f.find(".restore") != std::string::npos) {
|
2022-12-15 23:48:50 +00:00
|
|
|
return target()->DeleteFile(f, opts, dbg);
|
2019-12-16 22:28:06 +00:00
|
|
|
}
|
2021-04-28 17:57:11 +00:00
|
|
|
// Rename the file instead of deletion to keep the history, and
|
|
|
|
// at the same time it is not visible to RocksDB.
|
2022-12-15 23:48:50 +00:00
|
|
|
return target()->RenameFile(f, f + "_renamed_", opts, dbg);
|
2019-12-16 22:28:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If true, all manifest files will not be delted in DeleteFile().
|
|
|
|
bool if_preserve_all_manifests = true;
|
|
|
|
};
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
2019-12-16 22:28:06 +00:00
|
|
|
#endif // GFLAGS
|