mirror of
https://github.com/facebook/rocksdb.git
synced 2024-12-01 07:15:51 +00:00
241253053a
Summary:
This PR fix the issue that deletion of obsolete files during DB::Open are not rate limited.
The root cause is slow deletion is disabled if trash/db size ratio exceeds the configured `max_trash_db_ratio` d610e14f93/include/rocksdb/sst_file_manager.h (L126) however, the current handling in DB::Open starts with tracking nothing but the obsolete files. This will make the ratio always look like it's 1.
In order for the deletion rate limiting logic to work properly, we should only start deleting files after `SstFileManager` has finished tracking the whole DB, so the main fix is to move these two places that attempts to delete file after the tracking are done: 1) the `DeleteScheduler::CleanupDirectory` call in `SanitizeOptions`, 2) the `DB::DeleteObsoleteFiles` call.
There are some other aesthetic changes like refactoring collecting all the DB paths into a function, rename `DBImp::DeleteUnreferencedSstFiles` to `DBImpl:: MaybeUpdateNextFileNumber` as it doesn't actually delete the files.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12590
Test Plan: Added unit test and verified with manual testing
Reviewed By: anand1976
Differential Revision: D56830519
Pulled By: jowlyzhang
fbshipit-source-id: 8a38a21b1ea11c5371924f2b88663648f7a17885
114 lines
4.6 KiB
C++
114 lines
4.6 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
#pragma once
|
|
#include <string>
|
|
|
|
#include "file/filename.h"
|
|
#include "options/db_options.h"
|
|
#include "rocksdb/env.h"
|
|
#include "rocksdb/file_system.h"
|
|
#include "rocksdb/sst_file_writer.h"
|
|
#include "rocksdb/statistics.h"
|
|
#include "rocksdb/status.h"
|
|
#include "rocksdb/system_clock.h"
|
|
#include "rocksdb/types.h"
|
|
#include "trace_replay/io_tracer.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
// use_fsync maps to options.use_fsync, which determines the way that
|
|
// the file is synced after copying.
|
|
IOStatus CopyFile(FileSystem* fs, const std::string& source,
|
|
Temperature src_temp_hint,
|
|
std::unique_ptr<WritableFileWriter>& dest_writer,
|
|
uint64_t size, bool use_fsync,
|
|
const std::shared_ptr<IOTracer>& io_tracer);
|
|
IOStatus CopyFile(FileSystem* fs, const std::string& source,
|
|
Temperature src_temp_hint, const std::string& destination,
|
|
Temperature dst_temp, uint64_t size, bool use_fsync,
|
|
const std::shared_ptr<IOTracer>& io_tracer);
|
|
inline IOStatus CopyFile(const std::shared_ptr<FileSystem>& fs,
|
|
const std::string& source, Temperature src_temp_hint,
|
|
const std::string& destination, Temperature dst_temp,
|
|
uint64_t size, bool use_fsync,
|
|
const std::shared_ptr<IOTracer>& io_tracer) {
|
|
return CopyFile(fs.get(), source, src_temp_hint, destination, dst_temp, size,
|
|
use_fsync, io_tracer);
|
|
}
|
|
IOStatus CreateFile(FileSystem* fs, const std::string& destination,
|
|
const std::string& contents, bool use_fsync);
|
|
|
|
inline IOStatus CreateFile(const std::shared_ptr<FileSystem>& fs,
|
|
const std::string& destination,
|
|
const std::string& contents, bool use_fsync) {
|
|
return CreateFile(fs.get(), destination, contents, use_fsync);
|
|
}
|
|
|
|
// Delete a DB file, if this file is a SST file or Blob file and SstFileManager
|
|
// is used, it should have already been tracked by SstFileManager via its
|
|
// `OnFileAdd` API before passing to this API to be deleted, to ensure
|
|
// SstFileManager and its DeleteScheduler are tracking DB size and trash size
|
|
// properly.
|
|
Status DeleteDBFile(const ImmutableDBOptions* db_options,
|
|
const std::string& fname, const std::string& path_to_sync,
|
|
const bool force_bg, const bool force_fg);
|
|
|
|
// TODO(hx235): pass the whole DBOptions intead of its individual fields
|
|
IOStatus GenerateOneFileChecksum(
|
|
FileSystem* fs, const std::string& file_path,
|
|
FileChecksumGenFactory* checksum_factory,
|
|
const std::string& requested_checksum_func_name, std::string* file_checksum,
|
|
std::string* file_checksum_func_name,
|
|
size_t verify_checksums_readahead_size, bool allow_mmap_reads,
|
|
std::shared_ptr<IOTracer>& io_tracer, RateLimiter* rate_limiter,
|
|
const ReadOptions& read_options, Statistics* stats, SystemClock* clock);
|
|
|
|
inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro,
|
|
SystemClock* clock, IOOptions& opts) {
|
|
if (ro.deadline.count()) {
|
|
std::chrono::microseconds now =
|
|
std::chrono::microseconds(clock->NowMicros());
|
|
// Ensure there is atleast 1us available. We don't want to pass a value of
|
|
// 0 as that means no timeout
|
|
if (now >= ro.deadline) {
|
|
return IOStatus::TimedOut("Deadline exceeded");
|
|
}
|
|
opts.timeout = ro.deadline - now;
|
|
}
|
|
|
|
if (ro.io_timeout.count() &&
|
|
(!opts.timeout.count() || ro.io_timeout < opts.timeout)) {
|
|
opts.timeout = ro.io_timeout;
|
|
}
|
|
|
|
opts.rate_limiter_priority = ro.rate_limiter_priority;
|
|
opts.io_activity = ro.io_activity;
|
|
|
|
return IOStatus::OK();
|
|
}
|
|
|
|
inline IOStatus PrepareIOFromWriteOptions(const WriteOptions& wo,
|
|
IOOptions& opts) {
|
|
opts.rate_limiter_priority = wo.rate_limiter_priority;
|
|
opts.io_activity = wo.io_activity;
|
|
|
|
return IOStatus::OK();
|
|
}
|
|
|
|
// Test method to delete the input directory and all of its contents.
|
|
// This method is destructive and is meant for use only in tests!!!
|
|
Status DestroyDir(Env* env, const std::string& dir);
|
|
|
|
inline bool CheckFSFeatureSupport(FileSystem* fs, FSSupportedOps feat) {
|
|
int64_t supported_ops = 0;
|
|
fs->SupportedOps(supported_ops);
|
|
if (supported_ops & (1ULL << feat)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|