mirror of
https://github.com/facebook/rocksdb.git
synced 2024-12-01 16:15:59 +00:00
151242ce46
Summary: **Context:** The existing stat rocksdb.sst.read.micros does not reflect each of compaction and flush cases but aggregate them, which is not so helpful for us to understand IO read behavior of each of them. **Summary** - Update `StopWatch` and `RandomAccessFileReader` to record `rocksdb.sst.read.micros` and `rocksdb.file.{flush/compaction}.read.micros` - Fixed the default histogram in `RandomAccessFileReader` - New field `ReadOptions/IOOptions::io_activity`; Pass `ReadOptions` through paths under db open, flush and compaction to where we can prepare `IOOptions` and pass it to `RandomAccessFileReader` - Use `thread_status_util` for assertion in `DbStressFSWrapper` for continuous testing on we are passing correct `io_activity` under db open, flush and compaction Pull Request resolved: https://github.com/facebook/rocksdb/pull/11288 Test Plan: - **Stress test** - **Db bench 1: rocksdb.sst.read.micros COUNT ≈ sum of rocksdb.file.read.flush.micros's and rocksdb.file.read.compaction.micros's.** (without blob) - May not be exactly the same due to `HistogramStat::Add` only guarantees atomic not accuracy across threads. ``` ./db_bench -db=/dev/shm/testdb/ -statistics=true -benchmarks="fillseq" -key_size=32 -value_size=512 -num=50000 -write_buffer_size=655 -target_file_size_base=655 -disable_auto_compactions=false -compression_type=none -bloom_bits=3 (-use_plain_table=1 -prefix_size=10) ``` ``` // BlockBasedTable rocksdb.sst.read.micros P50 : 2.009374 P95 : 4.968548 P99 : 8.110362 P100 : 43.000000 COUNT : 40456 SUM : 114805 rocksdb.file.read.flush.micros P50 : 1.871841 P95 : 3.872407 P99 : 5.540541 P100 : 43.000000 COUNT : 2250 SUM : 6116 rocksdb.file.read.compaction.micros P50 : 2.023109 P95 : 5.029149 P99 : 8.196910 P100 : 26.000000 COUNT : 38206 SUM : 108689 // PlainTable Does not apply ``` - **Db bench 2: performance** **Read** SETUP: db with 900 files ``` ./db_bench -db=/dev/shm/testdb/ -benchmarks="fillseq" -key_size=32 -value_size=512 -num=50000 -write_buffer_size=655 -disable_auto_compactions=true -target_file_size_base=655 -compression_type=none ```run till convergence ``` ./db_bench -seed=1678564177044286 -use_existing_db=true -db=/dev/shm/testdb -benchmarks=readrandom[-X60] -statistics=true -num=1000000 -disable_auto_compactions=true -compression_type=none -bloom_bits=3 ``` Pre-change `readrandom [AVG 60 runs] : 21568 (± 248) ops/sec` Post-change (no regression, -0.3%) `readrandom [AVG 60 runs] : 21486 (± 236) ops/sec` **Compaction/Flush**run till convergence ``` ./db_bench -db=/dev/shm/testdb2/ -seed=1678564177044286 -benchmarks="fillseq[-X60]" -key_size=32 -value_size=512 -num=50000 -write_buffer_size=655 -disable_auto_compactions=false -target_file_size_base=655 -compression_type=none rocksdb.sst.read.micros COUNT : 33820 rocksdb.sst.read.flush.micros COUNT : 1800 rocksdb.sst.read.compaction.micros COUNT : 32020 ``` Pre-change `fillseq [AVG 46 runs] : 1391 (± 214) ops/sec; 0.7 (± 0.1) MB/sec` Post-change (no regression, ~-0.4%) `fillseq [AVG 46 runs] : 1385 (± 216) ops/sec; 0.7 (± 0.1) MB/sec` Reviewed By: ajkr Differential Revision: D44007011 Pulled By: hx235 fbshipit-source-id: a54c89e4846dfc9a135389edf3f3eedfea257132
140 lines
5.3 KiB
C++
140 lines
5.3 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#include <string>
|
|
|
|
#include "monitoring/thread_status_updater.h"
|
|
#include "rocksdb/db.h"
|
|
#include "rocksdb/env.h"
|
|
#include "rocksdb/thread_status.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
class ColumnFamilyData;
|
|
|
|
// The static utility class for updating thread-local status.
|
|
//
|
|
// The thread-local status is updated via the thread-local cached
|
|
// pointer thread_updater_local_cache_. During each function call,
|
|
// when ThreadStatusUtil finds thread_updater_local_cache_ is
|
|
// left uninitialized (determined by thread_updater_initialized_),
|
|
// it will tries to initialize it using the return value of
|
|
// Env::GetThreadStatusUpdater(). When thread_updater_local_cache_
|
|
// is initialized by a non-null pointer, each function call will
|
|
// then update the status of the current thread. Otherwise,
|
|
// all function calls to ThreadStatusUtil will be no-op.
|
|
class ThreadStatusUtil {
|
|
public:
|
|
// Register the current thread for tracking.
|
|
static void RegisterThread(const Env* env,
|
|
ThreadStatus::ThreadType thread_type);
|
|
|
|
// Unregister the current thread.
|
|
static void UnregisterThread();
|
|
|
|
// Create an entry in the global ColumnFamilyInfo table for the
|
|
// specified column family. This function should be called only
|
|
// when the current thread does not hold db_mutex.
|
|
static void NewColumnFamilyInfo(const DB* db, const ColumnFamilyData* cfd,
|
|
const std::string& cf_name, const Env* env);
|
|
|
|
// Erase the ConstantColumnFamilyInfo that is associated with the
|
|
// specified ColumnFamilyData. This function should be called only
|
|
// when the current thread does not hold db_mutex.
|
|
static void EraseColumnFamilyInfo(const ColumnFamilyData* cfd);
|
|
|
|
// Erase all ConstantColumnFamilyInfo that is associated with the
|
|
// specified db instance. This function should be called only when
|
|
// the current thread does not hold db_mutex.
|
|
static void EraseDatabaseInfo(const DB* db);
|
|
|
|
static void SetEnableTracking(bool enable_tracking);
|
|
|
|
// Update the thread status to indicate the current thread is doing
|
|
// something related to the specified column family.
|
|
//
|
|
// REQUIRES: cfd != nullptr
|
|
static void SetColumnFamily(const ColumnFamilyData* cfd);
|
|
|
|
static void SetThreadOperation(ThreadStatus::OperationType type);
|
|
|
|
static ThreadStatus::OperationType GetThreadOperation();
|
|
|
|
static ThreadStatus::OperationStage SetThreadOperationStage(
|
|
ThreadStatus::OperationStage stage);
|
|
|
|
static void SetThreadOperationProperty(int code, uint64_t value);
|
|
|
|
static void IncreaseThreadOperationProperty(int code, uint64_t delta);
|
|
|
|
static void SetThreadState(ThreadStatus::StateType type);
|
|
|
|
static void ResetThreadStatus();
|
|
|
|
#ifndef NDEBUG
|
|
static void TEST_SetStateDelay(const ThreadStatus::StateType state,
|
|
int micro);
|
|
static void TEST_StateDelay(const ThreadStatus::StateType state);
|
|
|
|
static Env::IOActivity TEST_GetExpectedIOActivity(
|
|
ThreadStatus::OperationType thread_op);
|
|
#endif
|
|
|
|
protected:
|
|
// Initialize the thread-local ThreadStatusUpdater when it finds
|
|
// the cached value is nullptr. Returns true if it has cached
|
|
// a non-null pointer.
|
|
static bool MaybeInitThreadLocalUpdater(const Env* env);
|
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
|
// A boolean flag indicating whether thread_updater_local_cache_
|
|
// is initialized. It is set to true when an Env uses any
|
|
// ThreadStatusUtil functions using the current thread other
|
|
// than UnregisterThread(). It will be set to false when
|
|
// UnregisterThread() is called.
|
|
//
|
|
// When this variable is set to true, thread_updater_local_cache_
|
|
// will not be updated until this variable is again set to false
|
|
// in UnregisterThread().
|
|
static thread_local bool thread_updater_initialized_;
|
|
|
|
// The thread-local cached ThreadStatusUpdater that caches the
|
|
// thread_status_updater_ of the first Env that uses any ThreadStatusUtil
|
|
// function other than UnregisterThread(). This variable will
|
|
// be cleared when UnregisterThread() is called.
|
|
//
|
|
// When this variable is set to a non-null pointer, then the status
|
|
// of the current thread will be updated when a function of
|
|
// ThreadStatusUtil is called. Otherwise, all functions of
|
|
// ThreadStatusUtil will be no-op.
|
|
//
|
|
// When thread_updater_initialized_ is set to true, this variable
|
|
// will not be updated until this thread_updater_initialized_ is
|
|
// again set to false in UnregisterThread().
|
|
static thread_local ThreadStatusUpdater* thread_updater_local_cache_;
|
|
#else
|
|
static bool thread_updater_initialized_;
|
|
static ThreadStatusUpdater* thread_updater_local_cache_;
|
|
#endif
|
|
};
|
|
|
|
// A helper class for updating thread state. It will set the
|
|
// thread state according to the input parameter in its constructor
|
|
// and set the thread state to the previous state in its destructor.
|
|
class AutoThreadOperationStageUpdater {
|
|
public:
|
|
explicit AutoThreadOperationStageUpdater(ThreadStatus::OperationStage stage);
|
|
~AutoThreadOperationStageUpdater();
|
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
|
private:
|
|
ThreadStatus::OperationStage prev_stage_;
|
|
#endif
|
|
};
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|