mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-26 07:30:54 +00:00
Export BackupEngine
Summary: Lots of clients have problems with using StackableDB interface. It's nice to have BackupableDB as a layer on top of DB, but not necessary. This diff exports BackupEngine, which can be used to create backups without forcing clients to use StackableDB interface. Test Plan: backupable_db_test Reviewers: dhruba, ljin, swk Reviewed By: ljin CC: leveldb, benj Differential Revision: https://reviews.facebook.net/D15477
This commit is contained in:
Igor Canadi
parent
9dc29414e3
commit
ec2fa4a690
|
|
@ -68,8 +68,6 @@ struct BackupableDBOptions {
|
|||
destroy_old_data(_destroy_old_data) { }
|
||||
};
|
||||
|
||||
class BackupEngine;
|
||||
|
||||
typedef uint32_t BackupID;
|
||||
|
||||
struct BackupInfo {
|
||||
|
|
@ -82,6 +80,29 @@ struct BackupInfo {
|
|||
: backup_id(_backup_id), timestamp(_timestamp), size(_size) {}
|
||||
};
|
||||
|
||||
// Please see the documentation in BackupableDB and RestoreBackupableDB
|
||||
class BackupEngine {
|
||||
public:
|
||||
virtual ~BackupEngine() {}
|
||||
|
||||
virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false) = 0;
|
||||
virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
|
||||
virtual Status DeleteBackup(BackupID backup_id) = 0;
|
||||
virtual void StopBackup() = 0;
|
||||
|
||||
virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
|
||||
virtual Status RestoreDBFromBackup(BackupID backup_id,
|
||||
const std::string& db_dir,
|
||||
const std::string& wal_dir) = 0;
|
||||
virtual Status RestoreDBFromLatestBackup(const std::string& db_dir,
|
||||
const std::string& wal_dir) = 0;
|
||||
|
||||
virtual void DeleteBackupsNewerThan(uint64_t sequence_number) = 0;
|
||||
};
|
||||
|
||||
extern BackupEngine* CreateNewBackupEngine(Env* db_env,
|
||||
const BackupableDBOptions& options);
|
||||
|
||||
// Stack your DB with BackupableDB to be able to backup the DB
|
||||
class BackupableDB : public StackableDB {
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@
|
|||
|
||||
namespace rocksdb {
|
||||
|
||||
// -------- BackupEngine class ---------
|
||||
class BackupEngine {
|
||||
// -------- BackupEngineImpl class ---------
|
||||
class BackupEngineImpl : public BackupEngine {
|
||||
public:
|
||||
BackupEngine(Env* db_env, const BackupableDBOptions& options);
|
||||
~BackupEngine();
|
||||
BackupEngineImpl(Env* db_env, const BackupableDBOptions& options);
|
||||
~BackupEngineImpl();
|
||||
Status CreateNewBackup(DB* db, bool flush_before_backup = false);
|
||||
Status PurgeOldBackups(uint32_t num_backups_to_keep);
|
||||
Status DeleteBackup(BackupID backup_id);
|
||||
|
|
@ -188,7 +188,13 @@ class BackupEngine {
|
|||
static const size_t copy_file_buffer_size_ = 5 * 1024 * 1024LL; // 5MB
|
||||
};
|
||||
|
||||
BackupEngine::BackupEngine(Env* db_env, const BackupableDBOptions& options)
|
||||
BackupEngine* CreateNewBackupEngine(Env* db_env,
|
||||
const BackupableDBOptions& options) {
|
||||
return new BackupEngineImpl(db_env, options);
|
||||
}
|
||||
|
||||
BackupEngineImpl::BackupEngineImpl(Env* db_env,
|
||||
const BackupableDBOptions& options)
|
||||
: stop_backup_(false),
|
||||
options_(options),
|
||||
db_env_(db_env),
|
||||
|
|
@ -271,11 +277,9 @@ BackupEngine::BackupEngine(Env* db_env, const BackupableDBOptions& options)
|
|||
latest_backup_id_);
|
||||
}
|
||||
|
||||
BackupEngine::~BackupEngine() {
|
||||
LogFlush(options_.info_log);
|
||||
}
|
||||
BackupEngineImpl::~BackupEngineImpl() { LogFlush(options_.info_log); }
|
||||
|
||||
void BackupEngine::DeleteBackupsNewerThan(uint64_t sequence_number) {
|
||||
void BackupEngineImpl::DeleteBackupsNewerThan(uint64_t sequence_number) {
|
||||
for (auto backup : backups_) {
|
||||
if (backup.second.GetSequenceNumber() > sequence_number) {
|
||||
Log(options_.info_log,
|
||||
|
|
@ -295,7 +299,7 @@ void BackupEngine::DeleteBackupsNewerThan(uint64_t sequence_number) {
|
|||
GarbageCollection(false);
|
||||
}
|
||||
|
||||
Status BackupEngine::CreateNewBackup(DB* db, bool flush_before_backup) {
|
||||
Status BackupEngineImpl::CreateNewBackup(DB* db, bool flush_before_backup) {
|
||||
Status s;
|
||||
std::vector<std::string> live_files;
|
||||
VectorLogPtr live_wal_files;
|
||||
|
|
@ -405,7 +409,7 @@ Status BackupEngine::CreateNewBackup(DB* db, bool flush_before_backup) {
|
|||
return s;
|
||||
}
|
||||
|
||||
Status BackupEngine::PurgeOldBackups(uint32_t num_backups_to_keep) {
|
||||
Status BackupEngineImpl::PurgeOldBackups(uint32_t num_backups_to_keep) {
|
||||
Log(options_.info_log, "Purging old backups, keeping %u",
|
||||
num_backups_to_keep);
|
||||
while (num_backups_to_keep < backups_.size()) {
|
||||
|
|
@ -418,7 +422,7 @@ Status BackupEngine::PurgeOldBackups(uint32_t num_backups_to_keep) {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status BackupEngine::DeleteBackup(BackupID backup_id) {
|
||||
Status BackupEngineImpl::DeleteBackup(BackupID backup_id) {
|
||||
Log(options_.info_log, "Deleting backup %u", backup_id);
|
||||
auto backup = backups_.find(backup_id);
|
||||
if (backup == backups_.end()) {
|
||||
|
|
@ -431,7 +435,7 @@ Status BackupEngine::DeleteBackup(BackupID backup_id) {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
void BackupEngine::GetBackupInfo(std::vector<BackupInfo>* backup_info) {
|
||||
void BackupEngineImpl::GetBackupInfo(std::vector<BackupInfo>* backup_info) {
|
||||
backup_info->reserve(backups_.size());
|
||||
for (auto& backup : backups_) {
|
||||
if (!backup.second.Empty()) {
|
||||
|
|
@ -441,9 +445,9 @@ void BackupEngine::GetBackupInfo(std::vector<BackupInfo>* backup_info) {
|
|||
}
|
||||
}
|
||||
|
||||
Status BackupEngine::RestoreDBFromBackup(BackupID backup_id,
|
||||
const std::string &db_dir,
|
||||
const std::string &wal_dir) {
|
||||
Status BackupEngineImpl::RestoreDBFromBackup(BackupID backup_id,
|
||||
const std::string& db_dir,
|
||||
const std::string& wal_dir) {
|
||||
auto backup_itr = backups_.find(backup_id);
|
||||
if (backup_itr == backups_.end()) {
|
||||
return Status::NotFound("Backup not found");
|
||||
|
|
@ -517,7 +521,7 @@ Status BackupEngine::RestoreDBFromBackup(BackupID backup_id,
|
|||
}
|
||||
|
||||
// latest backup id is an ASCII representation of latest backup id
|
||||
Status BackupEngine::GetLatestBackupFileContents(uint32_t* latest_backup) {
|
||||
Status BackupEngineImpl::GetLatestBackupFileContents(uint32_t* latest_backup) {
|
||||
Status s;
|
||||
unique_ptr<SequentialFile> file;
|
||||
s = backup_env_->NewSequentialFile(GetLatestBackupFile(),
|
||||
|
|
@ -547,7 +551,7 @@ Status BackupEngine::GetLatestBackupFileContents(uint32_t* latest_backup) {
|
|||
// writing 4 bytes to the file is atomic alright, but we should *never*
|
||||
// do something like 1. delete file, 2. write new file
|
||||
// We write to a tmp file and then atomically rename
|
||||
Status BackupEngine::PutLatestBackupFileContents(uint32_t latest_backup) {
|
||||
Status BackupEngineImpl::PutLatestBackupFileContents(uint32_t latest_backup) {
|
||||
Status s;
|
||||
unique_ptr<WritableFile> file;
|
||||
EnvOptions env_options;
|
||||
|
|
@ -577,14 +581,11 @@ Status BackupEngine::PutLatestBackupFileContents(uint32_t latest_backup) {
|
|||
return s;
|
||||
}
|
||||
|
||||
Status BackupEngine::CopyFile(const std::string& src,
|
||||
const std::string& dst,
|
||||
Env* src_env,
|
||||
Env* dst_env,
|
||||
bool sync,
|
||||
uint64_t* size,
|
||||
uint32_t* checksum_value,
|
||||
uint64_t size_limit) {
|
||||
Status BackupEngineImpl::CopyFile(const std::string& src,
|
||||
const std::string& dst, Env* src_env,
|
||||
Env* dst_env, bool sync, uint64_t* size,
|
||||
uint32_t* checksum_value,
|
||||
uint64_t size_limit) {
|
||||
Status s;
|
||||
unique_ptr<WritableFile> dst_file;
|
||||
unique_ptr<SequentialFile> src_file;
|
||||
|
|
@ -644,12 +645,10 @@ Status BackupEngine::CopyFile(const std::string& src,
|
|||
}
|
||||
|
||||
// src_fname will always start with "/"
|
||||
Status BackupEngine::BackupFile(BackupID backup_id,
|
||||
BackupMeta* backup,
|
||||
bool shared,
|
||||
const std::string& src_dir,
|
||||
const std::string& src_fname,
|
||||
uint64_t size_limit) {
|
||||
Status BackupEngineImpl::BackupFile(BackupID backup_id, BackupMeta* backup,
|
||||
bool shared, const std::string& src_dir,
|
||||
const std::string& src_fname,
|
||||
uint64_t size_limit) {
|
||||
|
||||
assert(src_fname.size() > 0 && src_fname[0] == '/');
|
||||
std::string dst_relative = src_fname.substr(1);
|
||||
|
|
@ -697,10 +696,9 @@ Status BackupEngine::BackupFile(BackupID backup_id,
|
|||
return s;
|
||||
}
|
||||
|
||||
Status BackupEngine::CalculateChecksum(const std::string& src,
|
||||
Env* src_env,
|
||||
uint64_t size_limit,
|
||||
uint32_t* checksum_value) {
|
||||
Status BackupEngineImpl::CalculateChecksum(const std::string& src, Env* src_env,
|
||||
uint64_t size_limit,
|
||||
uint32_t* checksum_value) {
|
||||
*checksum_value = 0;
|
||||
if (size_limit == 0) {
|
||||
size_limit = std::numeric_limits<uint64_t>::max();
|
||||
|
|
@ -737,7 +735,7 @@ Status BackupEngine::CalculateChecksum(const std::string& src,
|
|||
return s;
|
||||
}
|
||||
|
||||
void BackupEngine::GarbageCollection(bool full_scan) {
|
||||
void BackupEngineImpl::GarbageCollection(bool full_scan) {
|
||||
Log(options_.info_log, "Starting garbage collection");
|
||||
std::vector<std::string> to_delete;
|
||||
for (auto& itr : backuped_file_infos_) {
|
||||
|
|
@ -817,7 +815,7 @@ void BackupEngine::GarbageCollection(bool full_scan) {
|
|||
|
||||
// ------- BackupMeta class --------
|
||||
|
||||
Status BackupEngine::BackupMeta::AddFile(const FileInfo& file_info) {
|
||||
Status BackupEngineImpl::BackupMeta::AddFile(const FileInfo& file_info) {
|
||||
size_ += file_info.size;
|
||||
files_.push_back(file_info.filename);
|
||||
|
||||
|
|
@ -840,7 +838,7 @@ Status BackupEngine::BackupMeta::AddFile(const FileInfo& file_info) {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
void BackupEngine::BackupMeta::Delete() {
|
||||
void BackupEngineImpl::BackupMeta::Delete() {
|
||||
for (const auto& file : files_) {
|
||||
auto itr = file_infos_->find(file);
|
||||
assert(itr != file_infos_->end());
|
||||
|
|
@ -860,7 +858,8 @@ void BackupEngine::BackupMeta::Delete() {
|
|||
// <file2> <crc32(literal string)> <crc32_value>
|
||||
// ...
|
||||
// TODO: maybe add checksum?
|
||||
Status BackupEngine::BackupMeta::LoadFromFile(const std::string& backup_dir) {
|
||||
Status BackupEngineImpl::BackupMeta::LoadFromFile(
|
||||
const std::string& backup_dir) {
|
||||
assert(Empty());
|
||||
Status s;
|
||||
unique_ptr<SequentialFile> backup_meta_file;
|
||||
|
|
@ -927,7 +926,7 @@ Status BackupEngine::BackupMeta::LoadFromFile(const std::string& backup_dir) {
|
|||
return s;
|
||||
}
|
||||
|
||||
Status BackupEngine::BackupMeta::StoreToFile(bool sync) {
|
||||
Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
|
||||
Status s;
|
||||
unique_ptr<WritableFile> backup_meta_file;
|
||||
EnvOptions env_options;
|
||||
|
|
@ -969,7 +968,8 @@ Status BackupEngine::BackupMeta::StoreToFile(bool sync) {
|
|||
// --- BackupableDB methods --------
|
||||
|
||||
BackupableDB::BackupableDB(DB* db, const BackupableDBOptions& options)
|
||||
: StackableDB(db), backup_engine_(new BackupEngine(db->GetEnv(), options)) {
|
||||
: StackableDB(db),
|
||||
backup_engine_(new BackupEngineImpl(db->GetEnv(), options)) {
|
||||
if (options.share_table_files) {
|
||||
backup_engine_->DeleteBackupsNewerThan(GetLatestSequenceNumber());
|
||||
}
|
||||
|
|
@ -1003,7 +1003,7 @@ void BackupableDB::StopBackup() {
|
|||
|
||||
RestoreBackupableDB::RestoreBackupableDB(Env* db_env,
|
||||
const BackupableDBOptions& options)
|
||||
: backup_engine_(new BackupEngine(db_env, options)) {}
|
||||
: backup_engine_(new BackupEngineImpl(db_env, options)) {}
|
||||
|
||||
RestoreBackupableDB::~RestoreBackupableDB() {
|
||||
delete backup_engine_;
|
||||
|
|
|
|||
Loading…
Reference in a new issue