mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-30 04:41:49 +00:00
0ed93552f4
Summary: This PR implements deletion of obsolete files in a follower RocksDB instance. The follower tails the leader's MANIFEST and creates links to newly added SST files. These links need to be deleted once those files become obsolete in order to reclaim space. There are three cases to be considered - 1. New files added and links created, but the Version could not be installed due to some missing files. Those links need to be preserved so a subsequent catch up attempt can succeed. We insert the next file number in the `VersionSet` to `pending_outputs_` to prevent their deletion. 2. Files deleted from the previous successfully installed `Version`. These are deleted as usual in `PurgeObsoleteFiles`. 3. New files added by a `VersionEdit` and deleted by a subsequent `VersionEdit`, both processed in the same catchup attempt. Links will be created for the new files when verifying a candidate `Version`. Those need to be deleted explicitly as they're never added to `VersionStorageInfo`, and thus not deleted by `PurgeObsoleteFiles`. Test plan - New unit tests in `db_follower_test`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12657 Reviewed By: jowlyzhang Differential Revision: D57462697 Pulled By: anand1976 fbshipit-source-id: 898f15570638dd4930f839ffd31c560f9cb73916
55 lines
1.7 KiB
C++
55 lines
1.7 KiB
C++
// Copyright (c) 2024-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "db/db_impl/db_impl.h"
|
|
#include "db/db_impl/db_impl_secondary.h"
|
|
#include "logging/logging.h"
|
|
#include "port/port.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
class DBImplFollower : public DBImplSecondary {
|
|
public:
|
|
DBImplFollower(const DBOptions& db_options, std::unique_ptr<Env>&& env,
|
|
const std::string& dbname, std::string src_path);
|
|
~DBImplFollower();
|
|
|
|
Status Close() override;
|
|
|
|
protected:
|
|
bool OwnTablesAndLogs() const override {
|
|
// TODO: Change this to true once we've properly implemented file
|
|
// deletion for the read scaling case
|
|
return true;
|
|
}
|
|
|
|
Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
bool /*readonly*/, bool /*error_if_wal_file_exists*/,
|
|
bool /*error_if_data_exists_in_wals*/,
|
|
bool /*is_retry*/ = false, uint64_t* = nullptr,
|
|
RecoveryContext* /*recovery_ctx*/ = nullptr,
|
|
bool* /*can_retry*/ = nullptr) override;
|
|
|
|
private:
|
|
friend class DB;
|
|
|
|
Status TryCatchUpWithLeader();
|
|
void PeriodicRefresh();
|
|
|
|
std::unique_ptr<Env> env_guard_;
|
|
std::unique_ptr<port::Thread> catch_up_thread_;
|
|
std::atomic<bool> stop_requested_;
|
|
std::string src_path_;
|
|
port::Mutex mu_;
|
|
port::CondVar cv_;
|
|
std::unique_ptr<std::list<uint64_t>::iterator> pending_outputs_inserted_elem_;
|
|
};
|
|
} // namespace ROCKSDB_NAMESPACE
|