mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-27 11:43:49 +00:00
4da34b97ee
Summary: ### Context: Background compactions and flush generate large reads and writes, and can be long running, especially for universal compaction. In some cases, this can impact foreground reads and writes by users. ### Solution User, Flush, and Compaction reads share some code path. For this task, we update the rate_limiter_priority in ReadOptions for code paths (e.g. FindTable (mainly in BlockBasedTable::Open()) and various iterators), and eventually update the rate_limiter_priority in IOOptions for FSRandomAccessFile. **This PR is for the Read path.** The **Read:** dynamic priority for different state are listed as follows: | State | Normal | Delayed | Stalled | | ----- | ------ | ------- | ------- | | Flush (verification read in BuildTable()) | IO_USER | IO_USER | IO_USER | | Compaction | IO_LOW | IO_USER | IO_USER | | User | User provided | User provided | User provided | We will respect the read_options that the user provided and will not set it. The only sst read for Flush is the verification read in BuildTable(). It claims to be "regard as user read". **Details** 1. Set read_options.rate_limiter_priority dynamically: - User: Do not update the read_options. Use the read_options that the user provided. - Compaction: Update read_options in CompactionJob::ProcessKeyValueCompaction(). - Flush: Update read_options in BuildTable(). 2. Pass the rate limiter priority to FSRandomAccessFile functions: - After calling the FindTable(), read_options is passed through GetTableReader(table_cache.cc), BlockBasedTableFactory::NewTableReader(block_based_table_factory.cc), and BlockBasedTable::Open(). The Open() needs some updates for the ReadOptions variable and the updates are also needed for the called functions, including PrefetchTail(), PrepareIOOptions(), ReadFooterFromFile(), ReadMetaIndexblock(), ReadPropertiesBlock(), PrefetchIndexAndFilterBlocks(), and ReadRangeDelBlock(). - In RandomAccessFileReader, the functions to be updated include Read(), MultiRead(), ReadAsync(), and Prefetch(). - Update the downstream functions of NewIndexIterator(), NewDataBlockIterator(), and BlockBasedTableIterator(). ### Test Plans Add unit tests. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9996 Reviewed By: anand1976 Differential Revision: D36452483 Pulled By: gitbw95 fbshipit-source-id: 60978204a4f849bb9261cb78d9bc1cb56d6008cf
90 lines
3.8 KiB
C++
90 lines
3.8 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
#pragma once
|
|
#include <string>
|
|
|
|
#include "file/filename.h"
|
|
#include "options/db_options.h"
|
|
#include "rocksdb/env.h"
|
|
#include "rocksdb/file_system.h"
|
|
#include "rocksdb/sst_file_writer.h"
|
|
#include "rocksdb/status.h"
|
|
#include "rocksdb/system_clock.h"
|
|
#include "rocksdb/types.h"
|
|
#include "trace_replay/io_tracer.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
// use_fsync maps to options.use_fsync, which determines the way that
|
|
// the file is synced after copying.
|
|
extern IOStatus CopyFile(FileSystem* fs, const std::string& source,
|
|
std::unique_ptr<WritableFileWriter>& dest_writer,
|
|
uint64_t size, bool use_fsync,
|
|
const std::shared_ptr<IOTracer>& io_tracer,
|
|
const Temperature temperature);
|
|
extern IOStatus CopyFile(FileSystem* fs, const std::string& source,
|
|
const std::string& destination, uint64_t size,
|
|
bool use_fsync,
|
|
const std::shared_ptr<IOTracer>& io_tracer,
|
|
const Temperature temperature);
|
|
inline IOStatus CopyFile(const std::shared_ptr<FileSystem>& fs,
|
|
const std::string& source,
|
|
const std::string& destination, uint64_t size,
|
|
bool use_fsync,
|
|
const std::shared_ptr<IOTracer>& io_tracer,
|
|
const Temperature temperature) {
|
|
return CopyFile(fs.get(), source, destination, size, use_fsync, io_tracer,
|
|
temperature);
|
|
}
|
|
extern IOStatus CreateFile(FileSystem* fs, const std::string& destination,
|
|
const std::string& contents, bool use_fsync);
|
|
|
|
inline IOStatus CreateFile(const std::shared_ptr<FileSystem>& fs,
|
|
const std::string& destination,
|
|
const std::string& contents, bool use_fsync) {
|
|
return CreateFile(fs.get(), destination, contents, use_fsync);
|
|
}
|
|
|
|
extern Status DeleteDBFile(const ImmutableDBOptions* db_options,
|
|
const std::string& fname,
|
|
const std::string& path_to_sync, const bool force_bg,
|
|
const bool force_fg);
|
|
|
|
extern IOStatus GenerateOneFileChecksum(
|
|
FileSystem* fs, const std::string& file_path,
|
|
FileChecksumGenFactory* checksum_factory,
|
|
const std::string& requested_checksum_func_name, std::string* file_checksum,
|
|
std::string* file_checksum_func_name,
|
|
size_t verify_checksums_readahead_size, bool allow_mmap_reads,
|
|
std::shared_ptr<IOTracer>& io_tracer, RateLimiter* rate_limiter,
|
|
Env::IOPriority rate_limiter_priority);
|
|
|
|
inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro,
|
|
SystemClock* clock, IOOptions& opts) {
|
|
if (ro.deadline.count()) {
|
|
std::chrono::microseconds now =
|
|
std::chrono::microseconds(clock->NowMicros());
|
|
// Ensure there is atleast 1us available. We don't want to pass a value of
|
|
// 0 as that means no timeout
|
|
if (now >= ro.deadline) {
|
|
return IOStatus::TimedOut("Deadline exceeded");
|
|
}
|
|
opts.timeout = ro.deadline - now;
|
|
}
|
|
|
|
if (ro.io_timeout.count() &&
|
|
(!opts.timeout.count() || ro.io_timeout < opts.timeout)) {
|
|
opts.timeout = ro.io_timeout;
|
|
}
|
|
|
|
opts.rate_limiter_priority = ro.rate_limiter_priority;
|
|
return IOStatus::OK();
|
|
}
|
|
|
|
// Test method to delete the input directory and all of its contents.
|
|
// This method is destructive and is meant for use only in tests!!!
|
|
Status DestroyDir(Env* env, const std::string& dir);
|
|
} // namespace ROCKSDB_NAMESPACE
|