2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2014-12-30 18:39:13 +00:00
|
|
|
//
|
|
|
|
// This file defines the structures for thread operation and state.
|
|
|
|
// Thread operations are used to describe high level action of a
|
|
|
|
// thread such as doing compaction or flush, while thread state
|
|
|
|
// are used to describe lower-level action such as reading /
|
|
|
|
// writing a file or waiting for a mutex. Operations and states
|
|
|
|
// are designed to be independent. Typically, a thread usually involves
|
|
|
|
// in one operation and one state at any specific point in time.
|
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <string>
|
|
|
|
|
2022-10-26 19:08:20 +00:00
|
|
|
#include "rocksdb/thread_status.h"
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2014-12-30 18:39:13 +00:00
|
|
|
|
2016-12-14 02:22:00 +00:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2014-12-30 18:39:13 +00:00
|
|
|
|
|
|
|
// The structure that describes a major thread operation.
|
|
|
|
struct OperationInfo {
|
|
|
|
const ThreadStatus::OperationType type;
|
|
|
|
const std::string name;
|
|
|
|
};
|
|
|
|
|
|
|
|
// The global operation table.
|
|
|
|
//
|
|
|
|
// When updating a status of a thread, the pointer of the OperationInfo
|
|
|
|
// of the current ThreadStatusData will be pointing to one of the
|
|
|
|
// rows in this global table.
|
|
|
|
//
|
|
|
|
// Note that it's not designed to be constant as in the future we
|
|
|
|
// might consider adding global count to the OperationInfo.
|
|
|
|
static OperationInfo global_operation_table[] = {
|
2022-10-26 19:08:20 +00:00
|
|
|
{ThreadStatus::OP_UNKNOWN, ""},
|
|
|
|
{ThreadStatus::OP_COMPACTION, "Compaction"},
|
2023-04-21 16:07:18 +00:00
|
|
|
{ThreadStatus::OP_FLUSH, "Flush"},
|
Group rocksdb.sst.read.micros stat by different user read IOActivity + misc (#11444)
Summary:
**Context/Summary:**
- Similar to https://github.com/facebook/rocksdb/pull/11288 but for user read such as `Get(), MultiGet(), DBIterator::XXX(), Verify(File)Checksum()`.
- For this, I refactored some user-facing `MultiGet` calls in `TransactionBase` and various types of `DB` so that it does not call a user-facing `Get()` but `GetImpl()` for passing the `ReadOptions::io_activity` check (see PR conversation)
- New user read stats breakdown are guarded by `kExceptDetailedTimers` since measurement shows they have 4-5% regression to the upstream/main.
- Misc
- More refactoring: with https://github.com/facebook/rocksdb/pull/11288, we complete passing `ReadOptions/IOOptions` to FS level. So we can now replace the previously [added](https://github.com/facebook/rocksdb/pull/9424) `rate_limiter_priority` parameter in `RandomAccessFileReader`'s `Read/MultiRead/Prefetch()` with `IOOptions::rate_limiter_priority`
- Also, `ReadAsync()` call time is measured in `SST_READ_MICRO` now
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11444
Test Plan:
- CI fake db crash/stress test
- Microbenchmarking
**Build** `make clean && ROCKSDB_NO_FBCODE=1 DEBUG_LEVEL=0 make -jN db_basic_bench`
- google benchmark version: https://github.com/google/benchmark/commit/604f6fd3f4b34a84ec4eb4db81d842fa4db829cd
- db_basic_bench_base: upstream
- db_basic_bench_pr: db_basic_bench_base + this PR
- asyncread_db_basic_bench_base: upstream + [db basic bench patch for IteratorNext](https://github.com/facebook/rocksdb/compare/main...hx235:rocksdb:micro_bench_async_read)
- asyncread_db_basic_bench_pr: asyncread_db_basic_bench_base + this PR
**Test**
Get
```
TEST_TMPDIR=/dev/shm ./db_basic_bench_{null_stat|base|pr} --benchmark_filter=DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/mmap:1/threads:1 --benchmark_repetitions=1000
```
Result
```
Coming soon
```
AsyncRead
```
TEST_TMPDIR=/dev/shm ./asyncread_db_basic_bench_{base|pr} --benchmark_filter=IteratorNext/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/async_io:1/include_detailed_timers:0 --benchmark_repetitions=1000 > syncread_db_basic_bench_{base|pr}.out
```
Result
```
Base:
1956,1956,1968,1977,1979,1986,1988,1988,1988,1990,1991,1991,1993,1993,1993,1993,1994,1996,1997,1997,1997,1998,1999,2001,2001,2002,2004,2007,2007,2008,
PR (2.3% regression, due to measuring `SST_READ_MICRO` that wasn't measured before):
1993,2014,2016,2022,2024,2027,2027,2028,2028,2030,2031,2031,2032,2032,2038,2039,2042,2044,2044,2047,2047,2047,2048,2049,2050,2052,2052,2052,2053,2053,
```
Reviewed By: ajkr
Differential Revision: D45918925
Pulled By: hx235
fbshipit-source-id: 58a54560d9ebeb3a59b6d807639692614dad058a
2023-08-09 00:26:50 +00:00
|
|
|
{ThreadStatus::OP_DBOPEN, "DBOpen"},
|
|
|
|
{ThreadStatus::OP_GET, "Get"},
|
|
|
|
{ThreadStatus::OP_MULTIGET, "MultiGet"},
|
|
|
|
{ThreadStatus::OP_DBITERATOR, "DBIterator"},
|
|
|
|
{ThreadStatus::OP_VERIFY_DB_CHECKSUM, "VerifyDBChecksum"},
|
|
|
|
{ThreadStatus::OP_VERIFY_FILE_CHECKSUMS, "VerifyFileChecksums"},
|
2023-09-15 15:30:44 +00:00
|
|
|
{ThreadStatus::OP_GETENTITY, "GetEntity"},
|
|
|
|
{ThreadStatus::OP_MULTIGETENTITY, "MultiGetEntity"},
|
Group rocksdb.sst.read.micros stat by different user read IOActivity + misc (#11444)
Summary:
**Context/Summary:**
- Similar to https://github.com/facebook/rocksdb/pull/11288 but for user read such as `Get(), MultiGet(), DBIterator::XXX(), Verify(File)Checksum()`.
- For this, I refactored some user-facing `MultiGet` calls in `TransactionBase` and various types of `DB` so that it does not call a user-facing `Get()` but `GetImpl()` for passing the `ReadOptions::io_activity` check (see PR conversation)
- New user read stats breakdown are guarded by `kExceptDetailedTimers` since measurement shows they have 4-5% regression to the upstream/main.
- Misc
- More refactoring: with https://github.com/facebook/rocksdb/pull/11288, we complete passing `ReadOptions/IOOptions` to FS level. So we can now replace the previously [added](https://github.com/facebook/rocksdb/pull/9424) `rate_limiter_priority` parameter in `RandomAccessFileReader`'s `Read/MultiRead/Prefetch()` with `IOOptions::rate_limiter_priority`
- Also, `ReadAsync()` call time is measured in `SST_READ_MICRO` now
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11444
Test Plan:
- CI fake db crash/stress test
- Microbenchmarking
**Build** `make clean && ROCKSDB_NO_FBCODE=1 DEBUG_LEVEL=0 make -jN db_basic_bench`
- google benchmark version: https://github.com/google/benchmark/commit/604f6fd3f4b34a84ec4eb4db81d842fa4db829cd
- db_basic_bench_base: upstream
- db_basic_bench_pr: db_basic_bench_base + this PR
- asyncread_db_basic_bench_base: upstream + [db basic bench patch for IteratorNext](https://github.com/facebook/rocksdb/compare/main...hx235:rocksdb:micro_bench_async_read)
- asyncread_db_basic_bench_pr: asyncread_db_basic_bench_base + this PR
**Test**
Get
```
TEST_TMPDIR=/dev/shm ./db_basic_bench_{null_stat|base|pr} --benchmark_filter=DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/mmap:1/threads:1 --benchmark_repetitions=1000
```
Result
```
Coming soon
```
AsyncRead
```
TEST_TMPDIR=/dev/shm ./asyncread_db_basic_bench_{base|pr} --benchmark_filter=IteratorNext/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/async_io:1/include_detailed_timers:0 --benchmark_repetitions=1000 > syncread_db_basic_bench_{base|pr}.out
```
Result
```
Base:
1956,1956,1968,1977,1979,1986,1988,1988,1988,1990,1991,1991,1993,1993,1993,1993,1994,1996,1997,1997,1997,1998,1999,2001,2001,2002,2004,2007,2007,2008,
PR (2.3% regression, due to measuring `SST_READ_MICRO` that wasn't measured before):
1993,2014,2016,2022,2024,2027,2027,2028,2028,2030,2031,2031,2032,2032,2038,2039,2042,2044,2044,2047,2047,2047,2048,2049,2050,2052,2052,2052,2053,2053,
```
Reviewed By: ajkr
Differential Revision: D45918925
Pulled By: hx235
fbshipit-source-id: 58a54560d9ebeb3a59b6d807639692614dad058a
2023-08-09 00:26:50 +00:00
|
|
|
|
|
|
|
};
|
2014-12-30 18:39:13 +00:00
|
|
|
|
2015-03-13 17:45:40 +00:00
|
|
|
struct OperationStageInfo {
|
|
|
|
const ThreadStatus::OperationStage stage;
|
|
|
|
const std::string name;
|
|
|
|
};
|
|
|
|
|
|
|
|
// A table maintains the mapping from stage type to stage string.
|
|
|
|
// Note that the string must be changed accordingly when the
|
|
|
|
// associated function name changed.
|
|
|
|
static OperationStageInfo global_op_stage_table[] = {
|
2022-10-26 19:08:20 +00:00
|
|
|
{ThreadStatus::STAGE_UNKNOWN, ""},
|
|
|
|
{ThreadStatus::STAGE_FLUSH_RUN, "FlushJob::Run"},
|
|
|
|
{ThreadStatus::STAGE_FLUSH_WRITE_L0, "FlushJob::WriteLevel0Table"},
|
|
|
|
{ThreadStatus::STAGE_COMPACTION_PREPARE, "CompactionJob::Prepare"},
|
|
|
|
{ThreadStatus::STAGE_COMPACTION_RUN, "CompactionJob::Run"},
|
|
|
|
{ThreadStatus::STAGE_COMPACTION_PROCESS_KV,
|
|
|
|
"CompactionJob::ProcessKeyValueCompaction"},
|
|
|
|
{ThreadStatus::STAGE_COMPACTION_INSTALL, "CompactionJob::Install"},
|
|
|
|
{ThreadStatus::STAGE_COMPACTION_SYNC_FILE,
|
|
|
|
"CompactionJob::FinishCompactionOutputFile"},
|
|
|
|
{ThreadStatus::STAGE_PICK_MEMTABLES_TO_FLUSH,
|
|
|
|
"MemTableList::PickMemtablesToFlush"},
|
|
|
|
{ThreadStatus::STAGE_MEMTABLE_ROLLBACK,
|
|
|
|
"MemTableList::RollbackMemtableFlush"},
|
|
|
|
{ThreadStatus::STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS,
|
|
|
|
"MemTableList::TryInstallMemtableFlushResults"},
|
2015-03-13 17:45:40 +00:00
|
|
|
};
|
|
|
|
|
2014-12-30 18:39:13 +00:00
|
|
|
// The structure that describes a state.
|
|
|
|
struct StateInfo {
|
|
|
|
const ThreadStatus::StateType type;
|
|
|
|
const std::string name;
|
|
|
|
};
|
|
|
|
|
|
|
|
// The global state table.
|
|
|
|
//
|
|
|
|
// When updating a status of a thread, the pointer of the StateInfo
|
|
|
|
// of the current ThreadStatusData will be pointing to one of the
|
|
|
|
// rows in this global table.
|
|
|
|
static StateInfo global_state_table[] = {
|
2022-10-26 19:08:20 +00:00
|
|
|
{ThreadStatus::STATE_UNKNOWN, ""},
|
|
|
|
{ThreadStatus::STATE_MUTEX_WAIT, "Mutex Wait"},
|
2014-12-30 18:39:13 +00:00
|
|
|
};
|
|
|
|
|
2015-05-07 05:50:35 +00:00
|
|
|
struct OperationProperty {
|
|
|
|
int code;
|
|
|
|
std::string name;
|
|
|
|
};
|
|
|
|
|
|
|
|
static OperationProperty compaction_operation_properties[] = {
|
2022-10-26 19:08:20 +00:00
|
|
|
{ThreadStatus::COMPACTION_JOB_ID, "JobID"},
|
|
|
|
{ThreadStatus::COMPACTION_INPUT_OUTPUT_LEVEL, "InputOutputLevel"},
|
|
|
|
{ThreadStatus::COMPACTION_PROP_FLAGS, "Manual/Deletion/Trivial"},
|
|
|
|
{ThreadStatus::COMPACTION_TOTAL_INPUT_BYTES, "TotalInputBytes"},
|
|
|
|
{ThreadStatus::COMPACTION_BYTES_READ, "BytesRead"},
|
|
|
|
{ThreadStatus::COMPACTION_BYTES_WRITTEN, "BytesWritten"},
|
2015-05-07 05:50:35 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static OperationProperty flush_operation_properties[] = {
|
2022-10-26 19:08:20 +00:00
|
|
|
{ThreadStatus::FLUSH_JOB_ID, "JobID"},
|
|
|
|
{ThreadStatus::FLUSH_BYTES_MEMTABLES, "BytesMemtables"},
|
|
|
|
{ThreadStatus::FLUSH_BYTES_WRITTEN, "BytesWritten"}};
|
2015-05-07 05:50:35 +00:00
|
|
|
|
2014-12-30 18:39:13 +00:00
|
|
|
#else
|
|
|
|
|
2022-10-26 19:08:20 +00:00
|
|
|
struct OperationInfo {};
|
2014-12-30 18:39:13 +00:00
|
|
|
|
2022-10-26 19:08:20 +00:00
|
|
|
struct StateInfo {};
|
2014-12-30 18:39:13 +00:00
|
|
|
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|