mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-30 13:41:46 +00:00
147f7b472a
Summary: If the corruption of a table file is done before flushing, then db manifest may record the checksum for the corrupted table, which results in "matching checksums" when backup engine tries to verfiy the checksum, and causes a flaky test. Fix the issue by adding `Flush()` before trying to corrupt a table file in *db*. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7082 Test Plan: `buck test` Without the fix, failed 5 of 100 tests. Suspected whether the pseudo randomness causes the issue: doubling `keys_iteration` resulted in 2 of 100 tests failed; deterministically corrupting tables file also caused 2 of 100 tests to fail. With the fix, passed 200 of 200 tests. Reviewed By: pdillinger Differential Revision: D22375421 Pulled By: gg814 fbshipit-source-id: 7304618e7520684b6087e42d0b58329c5ad18329
2369 lines
83 KiB
C++
2369 lines
83 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
|
|
|
|
#include "rocksdb/utilities/backupable_db.h"
|
|
|
|
#include <algorithm>
|
|
#include <limits>
|
|
#include <string>
|
|
#include <utility>
|
|
|
|
#include "db/db_impl/db_impl.h"
|
|
#include "env/env_chroot.h"
|
|
#include "file/filename.h"
|
|
#include "port/port.h"
|
|
#include "port/stack_trace.h"
|
|
#include "rocksdb/rate_limiter.h"
|
|
#include "rocksdb/transaction_log.h"
|
|
#include "rocksdb/types.h"
|
|
#include "rocksdb/utilities/options_util.h"
|
|
#include "test_util/sync_point.h"
|
|
#include "test_util/testharness.h"
|
|
#include "test_util/testutil.h"
|
|
#include "util/cast_util.h"
|
|
#include "util/mutexlock.h"
|
|
#include "util/random.h"
|
|
#include "util/stderr_logger.h"
|
|
#include "util/string_util.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
namespace {
|
|
|
|
class DummyDB : public StackableDB {
|
|
public:
|
|
/* implicit */
|
|
DummyDB(const Options& options, const std::string& dbname)
|
|
: StackableDB(nullptr), options_(options), dbname_(dbname),
|
|
deletions_enabled_(true), sequence_number_(0) {}
|
|
|
|
SequenceNumber GetLatestSequenceNumber() const override {
|
|
return ++sequence_number_;
|
|
}
|
|
|
|
const std::string& GetName() const override { return dbname_; }
|
|
|
|
Env* GetEnv() const override { return options_.env; }
|
|
|
|
using DB::GetOptions;
|
|
Options GetOptions(ColumnFamilyHandle* /*column_family*/) const override {
|
|
return options_;
|
|
}
|
|
|
|
DBOptions GetDBOptions() const override { return DBOptions(options_); }
|
|
|
|
Status EnableFileDeletions(bool /*force*/) override {
|
|
EXPECT_TRUE(!deletions_enabled_);
|
|
deletions_enabled_ = true;
|
|
return Status::OK();
|
|
}
|
|
|
|
Status DisableFileDeletions() override {
|
|
EXPECT_TRUE(deletions_enabled_);
|
|
deletions_enabled_ = false;
|
|
return Status::OK();
|
|
}
|
|
|
|
Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,
|
|
bool /*flush_memtable*/ = true) override {
|
|
EXPECT_TRUE(!deletions_enabled_);
|
|
vec = live_files_;
|
|
*mfs = 100;
|
|
return Status::OK();
|
|
}
|
|
|
|
ColumnFamilyHandle* DefaultColumnFamily() const override { return nullptr; }
|
|
|
|
class DummyLogFile : public LogFile {
|
|
public:
|
|
/* implicit */
|
|
DummyLogFile(const std::string& path, bool alive = true)
|
|
: path_(path), alive_(alive) {}
|
|
|
|
std::string PathName() const override { return path_; }
|
|
|
|
uint64_t LogNumber() const override {
|
|
// what business do you have calling this method?
|
|
ADD_FAILURE();
|
|
return 0;
|
|
}
|
|
|
|
WalFileType Type() const override {
|
|
return alive_ ? kAliveLogFile : kArchivedLogFile;
|
|
}
|
|
|
|
SequenceNumber StartSequence() const override {
|
|
// this seqnum guarantees the dummy file will be included in the backup
|
|
// as long as it is alive.
|
|
return kMaxSequenceNumber;
|
|
}
|
|
|
|
uint64_t SizeFileBytes() const override { return 0; }
|
|
|
|
private:
|
|
std::string path_;
|
|
bool alive_;
|
|
}; // DummyLogFile
|
|
|
|
Status GetSortedWalFiles(VectorLogPtr& files) override {
|
|
EXPECT_TRUE(!deletions_enabled_);
|
|
files.resize(wal_files_.size());
|
|
for (size_t i = 0; i < files.size(); ++i) {
|
|
files[i].reset(
|
|
new DummyLogFile(wal_files_[i].first, wal_files_[i].second));
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
// To avoid FlushWAL called on stacked db which is nullptr
|
|
Status FlushWAL(bool /*sync*/) override { return Status::OK(); }
|
|
|
|
std::vector<std::string> live_files_;
|
|
// pair<filename, alive?>
|
|
std::vector<std::pair<std::string, bool>> wal_files_;
|
|
private:
|
|
Options options_;
|
|
std::string dbname_;
|
|
bool deletions_enabled_;
|
|
mutable SequenceNumber sequence_number_;
|
|
}; // DummyDB
|
|
|
|
class TestEnv : public EnvWrapper {
|
|
public:
|
|
explicit TestEnv(Env* t) : EnvWrapper(t) {}
|
|
|
|
class DummySequentialFile : public SequentialFile {
|
|
public:
|
|
explicit DummySequentialFile(bool fail_reads)
|
|
: SequentialFile(), rnd_(5), fail_reads_(fail_reads) {}
|
|
Status Read(size_t n, Slice* result, char* scratch) override {
|
|
if (fail_reads_) {
|
|
return Status::IOError();
|
|
}
|
|
size_t read_size = (n > size_left) ? size_left : n;
|
|
for (size_t i = 0; i < read_size; ++i) {
|
|
scratch[i] = rnd_.Next() & 255;
|
|
}
|
|
*result = Slice(scratch, read_size);
|
|
size_left -= read_size;
|
|
return Status::OK();
|
|
}
|
|
|
|
Status Skip(uint64_t n) override {
|
|
size_left = (n > size_left) ? size_left - n : 0;
|
|
return Status::OK();
|
|
}
|
|
|
|
private:
|
|
size_t size_left = 200;
|
|
Random rnd_;
|
|
bool fail_reads_;
|
|
};
|
|
|
|
Status NewSequentialFile(const std::string& f,
|
|
std::unique_ptr<SequentialFile>* r,
|
|
const EnvOptions& options) override {
|
|
MutexLock l(&mutex_);
|
|
if (dummy_sequential_file_) {
|
|
r->reset(
|
|
new TestEnv::DummySequentialFile(dummy_sequential_file_fail_reads_));
|
|
return Status::OK();
|
|
} else {
|
|
Status s = EnvWrapper::NewSequentialFile(f, r, options);
|
|
if (s.ok()) {
|
|
if ((*r)->use_direct_io()) {
|
|
++num_direct_seq_readers_;
|
|
}
|
|
++num_seq_readers_;
|
|
}
|
|
return s;
|
|
}
|
|
}
|
|
|
|
Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
|
|
const EnvOptions& options) override {
|
|
MutexLock l(&mutex_);
|
|
written_files_.push_back(f);
|
|
if (limit_written_files_ <= 0) {
|
|
return Status::NotSupported("Sorry, can't do this");
|
|
}
|
|
limit_written_files_--;
|
|
Status s = EnvWrapper::NewWritableFile(f, r, options);
|
|
if (s.ok()) {
|
|
if ((*r)->use_direct_io()) {
|
|
++num_direct_writers_;
|
|
}
|
|
++num_writers_;
|
|
}
|
|
return s;
|
|
}
|
|
|
|
Status NewRandomAccessFile(const std::string& fname,
|
|
std::unique_ptr<RandomAccessFile>* result,
|
|
const EnvOptions& options) override {
|
|
MutexLock l(&mutex_);
|
|
Status s = EnvWrapper::NewRandomAccessFile(fname, result, options);
|
|
if (s.ok()) {
|
|
if ((*result)->use_direct_io()) {
|
|
++num_direct_rand_readers_;
|
|
}
|
|
++num_rand_readers_;
|
|
}
|
|
return s;
|
|
}
|
|
|
|
Status DeleteFile(const std::string& fname) override {
|
|
MutexLock l(&mutex_);
|
|
if (fail_delete_files_) {
|
|
return Status::IOError();
|
|
}
|
|
EXPECT_GT(limit_delete_files_, 0U);
|
|
limit_delete_files_--;
|
|
return EnvWrapper::DeleteFile(fname);
|
|
}
|
|
|
|
Status DeleteDir(const std::string& dirname) override {
|
|
MutexLock l(&mutex_);
|
|
if (fail_delete_files_) {
|
|
return Status::IOError();
|
|
}
|
|
return EnvWrapper::DeleteDir(dirname);
|
|
}
|
|
|
|
void AssertWrittenFiles(std::vector<std::string>& should_have_written) {
|
|
MutexLock l(&mutex_);
|
|
std::sort(should_have_written.begin(), should_have_written.end());
|
|
std::sort(written_files_.begin(), written_files_.end());
|
|
|
|
ASSERT_EQ(should_have_written, written_files_);
|
|
}
|
|
|
|
void ClearWrittenFiles() {
|
|
MutexLock l(&mutex_);
|
|
written_files_.clear();
|
|
}
|
|
|
|
void SetLimitWrittenFiles(uint64_t limit) {
|
|
MutexLock l(&mutex_);
|
|
limit_written_files_ = limit;
|
|
}
|
|
|
|
void SetLimitDeleteFiles(uint64_t limit) {
|
|
MutexLock l(&mutex_);
|
|
limit_delete_files_ = limit;
|
|
}
|
|
|
|
void SetDeleteFileFailure(bool fail) {
|
|
MutexLock l(&mutex_);
|
|
fail_delete_files_ = fail;
|
|
}
|
|
|
|
void SetDummySequentialFile(bool dummy_sequential_file) {
|
|
MutexLock l(&mutex_);
|
|
dummy_sequential_file_ = dummy_sequential_file;
|
|
}
|
|
void SetDummySequentialFileFailReads(bool dummy_sequential_file_fail_reads) {
|
|
MutexLock l(&mutex_);
|
|
dummy_sequential_file_fail_reads_ = dummy_sequential_file_fail_reads;
|
|
}
|
|
|
|
void SetGetChildrenFailure(bool fail) { get_children_failure_ = fail; }
|
|
Status GetChildren(const std::string& dir,
|
|
std::vector<std::string>* r) override {
|
|
if (get_children_failure_) {
|
|
return Status::IOError("SimulatedFailure");
|
|
}
|
|
return EnvWrapper::GetChildren(dir, r);
|
|
}
|
|
|
|
// Some test cases do not actually create the test files (e.g., see
|
|
// DummyDB::live_files_) - for those cases, we mock those files' attributes
|
|
// so CreateNewBackup() can get their attributes.
|
|
void SetFilenamesForMockedAttrs(const std::vector<std::string>& filenames) {
|
|
filenames_for_mocked_attrs_ = filenames;
|
|
}
|
|
Status GetChildrenFileAttributes(
|
|
const std::string& dir, std::vector<Env::FileAttributes>* r) override {
|
|
if (filenames_for_mocked_attrs_.size() > 0) {
|
|
for (const auto& filename : filenames_for_mocked_attrs_) {
|
|
r->push_back({dir + filename, 10 /* size_bytes */});
|
|
}
|
|
return Status::OK();
|
|
}
|
|
return EnvWrapper::GetChildrenFileAttributes(dir, r);
|
|
}
|
|
Status GetFileSize(const std::string& path, uint64_t* size_bytes) override {
|
|
if (filenames_for_mocked_attrs_.size() > 0) {
|
|
auto fname = path.substr(path.find_last_of('/'));
|
|
auto filename_iter = std::find(filenames_for_mocked_attrs_.begin(),
|
|
filenames_for_mocked_attrs_.end(), fname);
|
|
if (filename_iter != filenames_for_mocked_attrs_.end()) {
|
|
*size_bytes = 10;
|
|
return Status::OK();
|
|
}
|
|
return Status::NotFound(fname);
|
|
}
|
|
return EnvWrapper::GetFileSize(path, size_bytes);
|
|
}
|
|
|
|
void SetCreateDirIfMissingFailure(bool fail) {
|
|
create_dir_if_missing_failure_ = fail;
|
|
}
|
|
Status CreateDirIfMissing(const std::string& d) override {
|
|
if (create_dir_if_missing_failure_) {
|
|
return Status::IOError("SimulatedFailure");
|
|
}
|
|
return EnvWrapper::CreateDirIfMissing(d);
|
|
}
|
|
|
|
void SetNewDirectoryFailure(bool fail) { new_directory_failure_ = fail; }
|
|
Status NewDirectory(const std::string& name,
|
|
std::unique_ptr<Directory>* result) override {
|
|
if (new_directory_failure_) {
|
|
return Status::IOError("SimulatedFailure");
|
|
}
|
|
return EnvWrapper::NewDirectory(name, result);
|
|
}
|
|
|
|
void ClearFileOpenCounters() {
|
|
MutexLock l(&mutex_);
|
|
num_rand_readers_ = 0;
|
|
num_direct_rand_readers_ = 0;
|
|
num_seq_readers_ = 0;
|
|
num_direct_seq_readers_ = 0;
|
|
num_writers_ = 0;
|
|
num_direct_writers_ = 0;
|
|
}
|
|
|
|
int num_rand_readers() { return num_rand_readers_; }
|
|
int num_direct_rand_readers() { return num_direct_rand_readers_; }
|
|
int num_seq_readers() { return num_seq_readers_; }
|
|
int num_direct_seq_readers() { return num_direct_seq_readers_; }
|
|
int num_writers() { return num_writers_; }
|
|
int num_direct_writers() { return num_direct_writers_; }
|
|
|
|
private:
|
|
port::Mutex mutex_;
|
|
bool dummy_sequential_file_ = false;
|
|
bool dummy_sequential_file_fail_reads_ = false;
|
|
std::vector<std::string> written_files_;
|
|
std::vector<std::string> filenames_for_mocked_attrs_;
|
|
uint64_t limit_written_files_ = 1000000;
|
|
uint64_t limit_delete_files_ = 1000000;
|
|
bool fail_delete_files_ = false;
|
|
|
|
bool get_children_failure_ = false;
|
|
bool create_dir_if_missing_failure_ = false;
|
|
bool new_directory_failure_ = false;
|
|
|
|
// Keeps track of how many files of each type were successfully opened, and
|
|
// out of those, how many were opened with direct I/O.
|
|
std::atomic<int> num_rand_readers_;
|
|
std::atomic<int> num_direct_rand_readers_;
|
|
std::atomic<int> num_seq_readers_;
|
|
std::atomic<int> num_direct_seq_readers_;
|
|
std::atomic<int> num_writers_;
|
|
std::atomic<int> num_direct_writers_;
|
|
}; // TestEnv
|
|
|
|
class FileManager : public EnvWrapper {
|
|
public:
|
|
explicit FileManager(Env* t) : EnvWrapper(t), rnd_(5) {}
|
|
|
|
Status GetRandomFileInDir(const std::string& dir, std::string* fname,
|
|
uint64_t* fsize) {
|
|
std::vector<FileAttributes> children;
|
|
GetChildrenFileAttributes(dir, &children);
|
|
if (children.size() <= 2) { // . and ..
|
|
return Status::NotFound("Empty directory: " + dir);
|
|
}
|
|
assert(fname != nullptr);
|
|
while (true) {
|
|
int i = rnd_.Next() % children.size();
|
|
if (children[i].name != "." && children[i].name != "..") {
|
|
fname->assign(dir + "/" + children[i].name);
|
|
*fsize = children[i].size_bytes;
|
|
return Status::OK();
|
|
}
|
|
}
|
|
// should never get here
|
|
assert(false);
|
|
return Status::NotFound("");
|
|
}
|
|
|
|
Status DeleteRandomFileInDir(const std::string& dir) {
|
|
std::vector<std::string> children;
|
|
GetChildren(dir, &children);
|
|
if (children.size() <= 2) { // . and ..
|
|
return Status::NotFound("");
|
|
}
|
|
while (true) {
|
|
int i = rnd_.Next() % children.size();
|
|
if (children[i] != "." && children[i] != "..") {
|
|
return DeleteFile(dir + "/" + children[i]);
|
|
}
|
|
}
|
|
// should never get here
|
|
assert(false);
|
|
return Status::NotFound("");
|
|
}
|
|
|
|
Status AppendToRandomFileInDir(const std::string& dir,
|
|
const std::string& data) {
|
|
std::vector<std::string> children;
|
|
GetChildren(dir, &children);
|
|
if (children.size() <= 2) {
|
|
return Status::NotFound("");
|
|
}
|
|
while (true) {
|
|
int i = rnd_.Next() % children.size();
|
|
if (children[i] != "." && children[i] != "..") {
|
|
return WriteToFile(dir + "/" + children[i], data);
|
|
}
|
|
}
|
|
// should never get here
|
|
assert(false);
|
|
return Status::NotFound("");
|
|
}
|
|
|
|
Status CorruptFile(const std::string& fname, uint64_t bytes_to_corrupt) {
|
|
std::string file_contents;
|
|
Status s = ReadFileToString(this, fname, &file_contents);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
s = DeleteFile(fname);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
|
|
for (uint64_t i = 0; i < bytes_to_corrupt; ++i) {
|
|
std::string tmp;
|
|
test::RandomString(&rnd_, 1, &tmp);
|
|
file_contents[rnd_.Next() % file_contents.size()] = tmp[0];
|
|
}
|
|
return WriteToFile(fname, file_contents);
|
|
}
|
|
|
|
Status CorruptChecksum(const std::string& fname, bool appear_valid) {
|
|
std::string metadata;
|
|
Status s = ReadFileToString(this, fname, &metadata);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
s = DeleteFile(fname);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
|
|
auto pos = metadata.find("private");
|
|
if (pos == std::string::npos) {
|
|
return Status::Corruption("private file is expected");
|
|
}
|
|
pos = metadata.find(" crc32 ", pos + 6);
|
|
if (pos == std::string::npos) {
|
|
return Status::Corruption("checksum not found");
|
|
}
|
|
|
|
if (metadata.size() < pos + 7) {
|
|
return Status::Corruption("bad CRC32 checksum value");
|
|
}
|
|
|
|
if (appear_valid) {
|
|
if (metadata[pos + 8] == '\n') {
|
|
// single digit value, safe to insert one more digit
|
|
metadata.insert(pos + 8, 1, '0');
|
|
} else {
|
|
metadata.erase(pos + 8, 1);
|
|
}
|
|
} else {
|
|
metadata[pos + 7] = 'a';
|
|
}
|
|
|
|
return WriteToFile(fname, metadata);
|
|
}
|
|
|
|
Status WriteToFile(const std::string& fname, const std::string& data) {
|
|
std::unique_ptr<WritableFile> file;
|
|
EnvOptions env_options;
|
|
env_options.use_mmap_writes = false;
|
|
Status s = EnvWrapper::NewWritableFile(fname, &file, env_options);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
return file->Append(Slice(data));
|
|
}
|
|
|
|
private:
|
|
Random rnd_;
|
|
}; // FileManager
|
|
|
|
// utility functions
|
|
static size_t FillDB(DB* db, int from, int to) {
|
|
size_t bytes_written = 0;
|
|
for (int i = from; i < to; ++i) {
|
|
std::string key = "testkey" + ToString(i);
|
|
std::string value = "testvalue" + ToString(i);
|
|
bytes_written += key.size() + value.size();
|
|
|
|
EXPECT_OK(db->Put(WriteOptions(), Slice(key), Slice(value)));
|
|
}
|
|
return bytes_written;
|
|
}
|
|
|
|
static void AssertExists(DB* db, int from, int to) {
|
|
for (int i = from; i < to; ++i) {
|
|
std::string key = "testkey" + ToString(i);
|
|
std::string value;
|
|
Status s = db->Get(ReadOptions(), Slice(key), &value);
|
|
ASSERT_EQ(value, "testvalue" + ToString(i));
|
|
}
|
|
}
|
|
|
|
static void AssertEmpty(DB* db, int from, int to) {
|
|
for (int i = from; i < to; ++i) {
|
|
std::string key = "testkey" + ToString(i);
|
|
std::string value = "testvalue" + ToString(i);
|
|
|
|
Status s = db->Get(ReadOptions(), Slice(key), &value);
|
|
ASSERT_TRUE(s.IsNotFound());
|
|
}
|
|
}
|
|
|
|
class BackupableDBTest : public testing::Test {
|
|
public:
|
|
enum ShareOption {
|
|
kNoShare,
|
|
kShareNoChecksum,
|
|
kShareWithChecksum,
|
|
};
|
|
|
|
const std::vector<ShareOption> kAllShareOptions = {
|
|
kNoShare, kShareNoChecksum, kShareWithChecksum};
|
|
|
|
BackupableDBTest() {
|
|
// set up files
|
|
std::string db_chroot = test::PerThreadDBPath("backupable_db");
|
|
std::string backup_chroot = test::PerThreadDBPath("backupable_db_backup");
|
|
Env::Default()->CreateDir(db_chroot);
|
|
Env::Default()->CreateDir(backup_chroot);
|
|
dbname_ = "/tempdb";
|
|
backupdir_ = "/tempbk";
|
|
|
|
// set up envs
|
|
db_chroot_env_.reset(NewChrootEnv(Env::Default(), db_chroot));
|
|
backup_chroot_env_.reset(NewChrootEnv(Env::Default(), backup_chroot));
|
|
test_db_env_.reset(new TestEnv(db_chroot_env_.get()));
|
|
test_backup_env_.reset(new TestEnv(backup_chroot_env_.get()));
|
|
file_manager_.reset(new FileManager(backup_chroot_env_.get()));
|
|
|
|
// set up db options
|
|
options_.create_if_missing = true;
|
|
options_.paranoid_checks = true;
|
|
options_.write_buffer_size = 1 << 17; // 128KB
|
|
options_.env = test_db_env_.get();
|
|
options_.wal_dir = dbname_;
|
|
|
|
// Create logger
|
|
DBOptions logger_options;
|
|
logger_options.env = db_chroot_env_.get();
|
|
CreateLoggerFromOptions(dbname_, logger_options, &logger_);
|
|
|
|
// set up backup db options
|
|
backupable_options_.reset(new BackupableDBOptions(
|
|
backupdir_, test_backup_env_.get(), true, logger_.get(), true));
|
|
|
|
// most tests will use multi-threaded backups
|
|
backupable_options_->max_background_operations = 7;
|
|
|
|
// delete old files in db
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
DB* OpenDB() {
|
|
DB* db;
|
|
EXPECT_OK(DB::Open(options_, dbname_, &db));
|
|
return db;
|
|
}
|
|
|
|
void OpenDBAndBackupEngine(bool destroy_old_data = false, bool dummy = false,
|
|
ShareOption shared_option = kShareNoChecksum) {
|
|
// reset all the defaults
|
|
test_backup_env_->SetLimitWrittenFiles(1000000);
|
|
test_db_env_->SetLimitWrittenFiles(1000000);
|
|
test_db_env_->SetDummySequentialFile(dummy);
|
|
|
|
DB* db;
|
|
if (dummy) {
|
|
dummy_db_ = new DummyDB(options_, dbname_);
|
|
db = dummy_db_;
|
|
} else {
|
|
ASSERT_OK(DB::Open(options_, dbname_, &db));
|
|
}
|
|
db_.reset(db);
|
|
backupable_options_->destroy_old_data = destroy_old_data;
|
|
backupable_options_->share_table_files = shared_option != kNoShare;
|
|
backupable_options_->share_files_with_checksum =
|
|
shared_option == kShareWithChecksum;
|
|
BackupEngine* backup_engine;
|
|
ASSERT_OK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
backup_engine_.reset(backup_engine);
|
|
}
|
|
|
|
void CloseDBAndBackupEngine() {
|
|
db_.reset();
|
|
backup_engine_.reset();
|
|
}
|
|
|
|
void OpenBackupEngine() {
|
|
backupable_options_->destroy_old_data = false;
|
|
BackupEngine* backup_engine;
|
|
ASSERT_OK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
backup_engine_.reset(backup_engine);
|
|
}
|
|
|
|
void CloseBackupEngine() { backup_engine_.reset(nullptr); }
|
|
|
|
// restores backup backup_id and asserts the existence of
|
|
// [start_exist, end_exist> and not-existence of
|
|
// [end_exist, end>
|
|
//
|
|
// if backup_id == 0, it means restore from latest
|
|
// if end == 0, don't check AssertEmpty
|
|
void AssertBackupConsistency(BackupID backup_id, uint32_t start_exist,
|
|
uint32_t end_exist, uint32_t end = 0,
|
|
bool keep_log_files = false) {
|
|
RestoreOptions restore_options(keep_log_files);
|
|
bool opened_backup_engine = false;
|
|
if (backup_engine_.get() == nullptr) {
|
|
opened_backup_engine = true;
|
|
OpenBackupEngine();
|
|
}
|
|
if (backup_id > 0) {
|
|
ASSERT_OK(backup_engine_->RestoreDBFromBackup(backup_id, dbname_, dbname_,
|
|
restore_options));
|
|
} else {
|
|
ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_,
|
|
restore_options));
|
|
}
|
|
DB* db = OpenDB();
|
|
AssertExists(db, start_exist, end_exist);
|
|
if (end != 0) {
|
|
AssertEmpty(db, end_exist, end);
|
|
}
|
|
delete db;
|
|
if (opened_backup_engine) {
|
|
CloseBackupEngine();
|
|
}
|
|
}
|
|
|
|
void DeleteLogFiles() {
|
|
std::vector<std::string> delete_logs;
|
|
db_chroot_env_->GetChildren(dbname_, &delete_logs);
|
|
for (auto f : delete_logs) {
|
|
uint64_t number;
|
|
FileType type;
|
|
bool ok = ParseFileName(f, &number, &type);
|
|
if (ok && type == kLogFile) {
|
|
db_chroot_env_->DeleteFile(dbname_ + "/" + f);
|
|
}
|
|
}
|
|
}
|
|
|
|
Status CorruptRandomTableFileInDB() {
|
|
Random rnd(6);
|
|
std::vector<FileAttributes> children;
|
|
test_db_env_->GetChildrenFileAttributes(dbname_, &children);
|
|
if (children.size() <= 2) { // . and ..
|
|
return Status::NotFound("");
|
|
}
|
|
std::string fname;
|
|
uint64_t fsize = 0;
|
|
while (true) {
|
|
int i = rnd.Next() % children.size();
|
|
fname = children[i].name;
|
|
fsize = children[i].size_bytes;
|
|
// find an sst file
|
|
if (fsize > 0 && fname.length() > 4 &&
|
|
fname.rfind(".sst") == fname.length() - 4) {
|
|
fname = dbname_ + "/" + fname;
|
|
break;
|
|
}
|
|
}
|
|
|
|
std::string file_contents;
|
|
Status s = ReadFileToString(test_db_env_.get(), fname, &file_contents);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
s = test_db_env_->DeleteFile(fname);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
for (uint64_t i = 0; i < fsize; ++i) {
|
|
std::string tmp;
|
|
test::RandomString(&rnd, 1, &tmp);
|
|
file_contents[rnd.Next() % file_contents.size()] = tmp[0];
|
|
}
|
|
return WriteStringToFile(test_db_env_.get(), file_contents, fname);
|
|
}
|
|
|
|
// files
|
|
std::string dbname_;
|
|
std::string backupdir_;
|
|
|
|
// logger_ must be above backup_engine_ such that the engine's destructor,
|
|
// which uses a raw pointer to the logger, executes first.
|
|
std::shared_ptr<Logger> logger_;
|
|
|
|
// envs
|
|
std::unique_ptr<Env> db_chroot_env_;
|
|
std::unique_ptr<Env> backup_chroot_env_;
|
|
std::unique_ptr<TestEnv> test_db_env_;
|
|
std::unique_ptr<TestEnv> test_backup_env_;
|
|
std::unique_ptr<FileManager> file_manager_;
|
|
|
|
// all the dbs!
|
|
DummyDB* dummy_db_; // BackupableDB owns dummy_db_
|
|
std::unique_ptr<DB> db_;
|
|
std::unique_ptr<BackupEngine> backup_engine_;
|
|
|
|
// options
|
|
Options options_;
|
|
|
|
protected:
|
|
std::unique_ptr<BackupableDBOptions> backupable_options_;
|
|
}; // BackupableDBTest
|
|
|
|
void AppendPath(const std::string& path, std::vector<std::string>& v) {
|
|
for (auto& f : v) {
|
|
f = path + f;
|
|
}
|
|
}
|
|
|
|
class BackupableDBTestWithParam : public BackupableDBTest,
|
|
public testing::WithParamInterface<bool> {
|
|
public:
|
|
BackupableDBTestWithParam() {
|
|
backupable_options_->share_files_with_checksum = GetParam();
|
|
}
|
|
};
|
|
|
|
// This test verifies that the verifyBackup method correctly identifies
|
|
// invalid backups
|
|
TEST_P(BackupableDBTestWithParam, VerifyBackup) {
|
|
const int keys_iteration = 5000;
|
|
Status s;
|
|
OpenDBAndBackupEngine(true);
|
|
// create five backups
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenDBAndBackupEngine();
|
|
// ---------- case 1. - valid backup -----------
|
|
ASSERT_TRUE(backup_engine_->VerifyBackup(1).ok());
|
|
|
|
// ---------- case 2. - delete a file -----------i
|
|
file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/1");
|
|
ASSERT_TRUE(backup_engine_->VerifyBackup(1).IsNotFound());
|
|
|
|
// ---------- case 3. - corrupt a file -----------
|
|
std::string append_data = "Corrupting a random file";
|
|
file_manager_->AppendToRandomFileInDir(backupdir_ + "/private/2",
|
|
append_data);
|
|
ASSERT_TRUE(backup_engine_->VerifyBackup(2).IsCorruption());
|
|
|
|
// ---------- case 4. - invalid backup -----------
|
|
ASSERT_TRUE(backup_engine_->VerifyBackup(6).IsNotFound());
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
// open DB, write, close DB, backup, restore, repeat
|
|
TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
|
|
// has to be a big number, so that it triggers the memtable flush
|
|
const int keys_iteration = 5000;
|
|
const int max_key = keys_iteration * 4 + 10;
|
|
// first iter -- flush before backup
|
|
// second iter -- don't flush before backup
|
|
for (int iter = 0; iter < 2; ++iter) {
|
|
// delete old data
|
|
DestroyDB(dbname_, options_);
|
|
bool destroy_data = true;
|
|
|
|
// every iteration --
|
|
// 1. insert new data in the DB
|
|
// 2. backup the DB
|
|
// 3. destroy the db
|
|
// 4. restore the db, check everything is still there
|
|
for (int i = 0; i < 5; ++i) {
|
|
// in last iteration, put smaller amount of data,
|
|
int fill_up_to = std::min(keys_iteration * (i + 1), max_key);
|
|
// ---- insert new data and back up ----
|
|
OpenDBAndBackupEngine(destroy_data);
|
|
destroy_data = false;
|
|
FillDB(db_.get(), keys_iteration * i, fill_up_to);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0));
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
|
|
// ---- make sure it's empty ----
|
|
DB* db = OpenDB();
|
|
AssertEmpty(db, 0, fill_up_to);
|
|
delete db;
|
|
|
|
// ---- restore the DB ----
|
|
OpenBackupEngine();
|
|
if (i >= 3) { // test purge old backups
|
|
// when i == 4, purge to only 1 backup
|
|
// when i == 3, purge to 2 backups
|
|
ASSERT_OK(backup_engine_->PurgeOldBackups(5 - i));
|
|
}
|
|
// ---- make sure the data is there ---
|
|
AssertBackupConsistency(0, 0, fill_up_to, max_key);
|
|
CloseBackupEngine();
|
|
}
|
|
}
|
|
}
|
|
|
|
// open DB, write, backup, write, backup, close, restore
|
|
TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
|
|
// has to be a big number, so that it triggers the memtable flush
|
|
const int keys_iteration = 5000;
|
|
const int max_key = keys_iteration * 4 + 10;
|
|
Random rnd(7);
|
|
// delete old data
|
|
DestroyDB(dbname_, options_);
|
|
|
|
OpenDBAndBackupEngine(true);
|
|
// write some data, backup, repeat
|
|
for (int i = 0; i < 5; ++i) {
|
|
if (i == 4) {
|
|
// delete backup number 2, online delete!
|
|
ASSERT_OK(backup_engine_->DeleteBackup(2));
|
|
}
|
|
// in last iteration, put smaller amount of data,
|
|
// so that backups can share sst files
|
|
int fill_up_to = std::min(keys_iteration * (i + 1), max_key);
|
|
FillDB(db_.get(), keys_iteration * i, fill_up_to);
|
|
// we should get consistent results with flush_before_backup
|
|
// set to both true and false
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
|
|
}
|
|
// close and destroy
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
|
|
// ---- make sure it's empty ----
|
|
DB* db = OpenDB();
|
|
AssertEmpty(db, 0, max_key);
|
|
delete db;
|
|
|
|
// ---- restore every backup and verify all the data is there ----
|
|
OpenBackupEngine();
|
|
for (int i = 1; i <= 5; ++i) {
|
|
if (i == 2) {
|
|
// we deleted backup 2
|
|
Status s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_);
|
|
ASSERT_TRUE(!s.ok());
|
|
} else {
|
|
int fill_up_to = std::min(keys_iteration * i, max_key);
|
|
AssertBackupConsistency(i, 0, fill_up_to, max_key);
|
|
}
|
|
}
|
|
|
|
// delete some backups -- this should leave only backups 3 and 5 alive
|
|
ASSERT_OK(backup_engine_->DeleteBackup(4));
|
|
ASSERT_OK(backup_engine_->PurgeOldBackups(2));
|
|
|
|
std::vector<BackupInfo> backup_info;
|
|
backup_engine_->GetBackupInfo(&backup_info);
|
|
ASSERT_EQ(2UL, backup_info.size());
|
|
|
|
// check backup 3
|
|
AssertBackupConsistency(3, 0, 3 * keys_iteration, max_key);
|
|
// check backup 5
|
|
AssertBackupConsistency(5, 0, max_key);
|
|
|
|
CloseBackupEngine();
|
|
}
|
|
|
|
INSTANTIATE_TEST_CASE_P(BackupableDBTestWithParam, BackupableDBTestWithParam,
|
|
::testing::Bool());
|
|
|
|
// this will make sure that backup does not copy the same file twice
|
|
TEST_F(BackupableDBTest, NoDoubleCopy_And_AutoGC) {
|
|
OpenDBAndBackupEngine(true, true);
|
|
|
|
// should write 5 DB files + one meta file
|
|
test_backup_env_->SetLimitWrittenFiles(7);
|
|
test_backup_env_->ClearWrittenFiles();
|
|
test_db_env_->SetLimitWrittenFiles(0);
|
|
dummy_db_->live_files_ = {"/00010.sst", "/00011.sst", "/CURRENT",
|
|
"/MANIFEST-01"};
|
|
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
|
|
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
|
std::vector<std::string> should_have_written = {
|
|
"/shared/.00010.sst.tmp", "/shared/.00011.sst.tmp", "/private/1/CURRENT",
|
|
"/private/1/MANIFEST-01", "/private/1/00011.log", "/meta/.1.tmp"};
|
|
AppendPath(backupdir_, should_have_written);
|
|
test_backup_env_->AssertWrittenFiles(should_have_written);
|
|
|
|
char db_number = '1';
|
|
|
|
for (std::string other_sst : {"00015.sst", "00017.sst", "00019.sst"}) {
|
|
// should write 4 new DB files + one meta file
|
|
// should not write/copy 00010.sst, since it's already there!
|
|
test_backup_env_->SetLimitWrittenFiles(6);
|
|
test_backup_env_->ClearWrittenFiles();
|
|
|
|
dummy_db_->live_files_ = {"/00010.sst", "/" + other_sst, "/CURRENT",
|
|
"/MANIFEST-01"};
|
|
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
|
|
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
|
// should not open 00010.sst - it's already there
|
|
|
|
++db_number;
|
|
std::string private_dir = std::string("/private/") + db_number;
|
|
should_have_written = {
|
|
"/shared/." + other_sst + ".tmp", private_dir + "/CURRENT",
|
|
private_dir + "/MANIFEST-01", private_dir + "/00011.log",
|
|
std::string("/meta/.") + db_number + ".tmp"};
|
|
AppendPath(backupdir_, should_have_written);
|
|
test_backup_env_->AssertWrittenFiles(should_have_written);
|
|
}
|
|
|
|
ASSERT_OK(backup_engine_->DeleteBackup(1));
|
|
ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00010.sst"));
|
|
|
|
// 00011.sst was only in backup 1, should be deleted
|
|
ASSERT_EQ(Status::NotFound(),
|
|
test_backup_env_->FileExists(backupdir_ + "/shared/00011.sst"));
|
|
ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst"));
|
|
|
|
// MANIFEST file size should be only 100
|
|
uint64_t size = 0;
|
|
test_backup_env_->GetFileSize(backupdir_ + "/private/2/MANIFEST-01", &size);
|
|
ASSERT_EQ(100UL, size);
|
|
test_backup_env_->GetFileSize(backupdir_ + "/shared/00015.sst", &size);
|
|
ASSERT_EQ(200UL, size);
|
|
|
|
CloseBackupEngine();
|
|
|
|
//
|
|
// Now simulate incomplete delete by removing just meta
|
|
//
|
|
ASSERT_OK(test_backup_env_->DeleteFile(backupdir_ + "/meta/2"));
|
|
|
|
OpenBackupEngine();
|
|
|
|
// 1 appears to be removed, so
|
|
// 2 non-corrupt and 0 corrupt seen
|
|
std::vector<BackupInfo> backup_info;
|
|
std::vector<BackupID> corrupt_backup_ids;
|
|
backup_engine_->GetBackupInfo(&backup_info);
|
|
backup_engine_->GetCorruptedBackups(&corrupt_backup_ids);
|
|
ASSERT_EQ(2UL, backup_info.size());
|
|
ASSERT_EQ(0UL, corrupt_backup_ids.size());
|
|
|
|
// Keep the two we see, but this should suffice to purge unreferenced
|
|
// shared files from incomplete delete.
|
|
ASSERT_OK(backup_engine_->PurgeOldBackups(2));
|
|
|
|
// Make sure dangling sst file has been removed (somewhere along this
|
|
// process). GarbageCollect should not be needed.
|
|
ASSERT_EQ(Status::NotFound(),
|
|
test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst"));
|
|
ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00017.sst"));
|
|
ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00019.sst"));
|
|
|
|
// Now actually purge a good one
|
|
ASSERT_OK(backup_engine_->PurgeOldBackups(1));
|
|
|
|
ASSERT_EQ(Status::NotFound(),
|
|
test_backup_env_->FileExists(backupdir_ + "/shared/00017.sst"));
|
|
ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00019.sst"));
|
|
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
// test various kind of corruptions that may happen:
|
|
// 1. Not able to write a file for backup - that backup should fail,
|
|
// everything else should work
|
|
// 2. Corrupted backup meta file or missing backuped file - we should
|
|
// not be able to open that backup, but all other backups should be
|
|
// fine
|
|
// 3. Corrupted checksum value - if the checksum is not a valid uint32_t,
|
|
// db open should fail, otherwise, it aborts during the restore process.
|
|
TEST_F(BackupableDBTest, CorruptionsTest) {
|
|
const int keys_iteration = 5000;
|
|
Random rnd(6);
|
|
Status s;
|
|
|
|
OpenDBAndBackupEngine(true);
|
|
// create five backups
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
|
|
}
|
|
|
|
// ---------- case 1. - fail a write -----------
|
|
// try creating backup 6, but fail a write
|
|
FillDB(db_.get(), keys_iteration * 5, keys_iteration * 6);
|
|
test_backup_env_->SetLimitWrittenFiles(2);
|
|
// should fail
|
|
s = backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2));
|
|
ASSERT_TRUE(!s.ok());
|
|
test_backup_env_->SetLimitWrittenFiles(1000000);
|
|
// latest backup should have all the keys
|
|
CloseDBAndBackupEngine();
|
|
AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6);
|
|
|
|
// --------- case 2. corrupted backup meta or missing backuped file ----
|
|
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/5", 3));
|
|
// since 5 meta is now corrupted, latest backup should be 4
|
|
AssertBackupConsistency(0, 0, keys_iteration * 4, keys_iteration * 5);
|
|
OpenBackupEngine();
|
|
s = backup_engine_->RestoreDBFromBackup(5, dbname_, dbname_);
|
|
ASSERT_TRUE(!s.ok());
|
|
CloseBackupEngine();
|
|
ASSERT_OK(file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/4"));
|
|
// 4 is corrupted, 3 is the latest backup now
|
|
AssertBackupConsistency(0, 0, keys_iteration * 3, keys_iteration * 5);
|
|
OpenBackupEngine();
|
|
s = backup_engine_->RestoreDBFromBackup(4, dbname_, dbname_);
|
|
CloseBackupEngine();
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
// --------- case 3. corrupted checksum value ----
|
|
ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/3", false));
|
|
// checksum of backup 3 is an invalid value, this can be detected at
|
|
// db open time, and it reverts to the previous backup automatically
|
|
AssertBackupConsistency(0, 0, keys_iteration * 2, keys_iteration * 5);
|
|
// checksum of the backup 2 appears to be valid, this can cause checksum
|
|
// mismatch and abort restore process
|
|
ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/2", true));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
|
|
OpenBackupEngine();
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
|
|
s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_);
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
// make sure that no corrupt backups have actually been deleted!
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/1"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/3"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/4"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/5"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/1"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/2"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/3"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/4"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/5"));
|
|
|
|
// delete the corrupt backups and then make sure they're actually deleted
|
|
ASSERT_OK(backup_engine_->DeleteBackup(5));
|
|
ASSERT_OK(backup_engine_->DeleteBackup(4));
|
|
ASSERT_OK(backup_engine_->DeleteBackup(3));
|
|
ASSERT_OK(backup_engine_->DeleteBackup(2));
|
|
// Should not be needed anymore with auto-GC on DeleteBackup
|
|
//(void)backup_engine_->GarbageCollect();
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/meta/5"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/private/5"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/meta/4"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/private/4"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/meta/3"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/private/3"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/meta/2"));
|
|
ASSERT_EQ(Status::NotFound(),
|
|
file_manager_->FileExists(backupdir_ + "/private/2"));
|
|
|
|
CloseBackupEngine();
|
|
AssertBackupConsistency(0, 0, keys_iteration * 1, keys_iteration * 5);
|
|
|
|
// new backup should be 2!
|
|
OpenDBAndBackupEngine();
|
|
FillDB(db_.get(), keys_iteration * 1, keys_iteration * 2);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
|
|
CloseDBAndBackupEngine();
|
|
AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 5);
|
|
}
|
|
|
|
// Corrupt a file but maintain its size
|
|
TEST_F(BackupableDBTest, CorruptFileMaintainSize) {
|
|
const int keys_iteration = 5000;
|
|
Status s;
|
|
OpenDBAndBackupEngine(true);
|
|
// create a backup
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenDBAndBackupEngine();
|
|
// verify with file size
|
|
ASSERT_OK(backup_engine_->VerifyBackup(1, false));
|
|
// verify with file checksum
|
|
ASSERT_OK(backup_engine_->VerifyBackup(1, true));
|
|
|
|
std::string file_to_corrupt;
|
|
uint64_t file_size = 0;
|
|
// under normal circumstance, there should be at least one nonempty file
|
|
while (file_size == 0) {
|
|
// get a random file in /private/1
|
|
ASSERT_OK(file_manager_->GetRandomFileInDir(backupdir_ + "/private/1",
|
|
&file_to_corrupt, &file_size));
|
|
// corrupt the file by replacing its content by file_size random bytes
|
|
ASSERT_OK(file_manager_->CorruptFile(file_to_corrupt, file_size));
|
|
}
|
|
// file sizes match
|
|
ASSERT_OK(backup_engine_->VerifyBackup(1, false));
|
|
// file checksums mismatch
|
|
ASSERT_NOK(backup_engine_->VerifyBackup(1, true));
|
|
// sanity check, use default second argument
|
|
ASSERT_OK(backup_engine_->VerifyBackup(1));
|
|
CloseDBAndBackupEngine();
|
|
|
|
// an extra challenge
|
|
// set share_files_with_checksum to true and do two more backups
|
|
// corrupt all the table files in shared_checksum but maintain their sizes
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
|
|
kShareWithChecksum);
|
|
// creat two backups
|
|
for (int i = 1; i < 3; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenDBAndBackupEngine();
|
|
std::vector<FileAttributes> children;
|
|
const std::string dir = backupdir_ + "/shared_checksum";
|
|
ASSERT_OK(file_manager_->GetChildrenFileAttributes(dir, &children));
|
|
for (const auto& child : children) {
|
|
if (child.name == "." || child.name == ".." || child.size_bytes == 0) {
|
|
continue;
|
|
}
|
|
// corrupt the file by replacing its content by file_size random bytes
|
|
ASSERT_OK(
|
|
file_manager_->CorruptFile(dir + "/" + child.name, child.size_bytes));
|
|
}
|
|
// file sizes match
|
|
ASSERT_OK(backup_engine_->VerifyBackup(1, false));
|
|
ASSERT_OK(backup_engine_->VerifyBackup(2, false));
|
|
// file checksums mismatch
|
|
ASSERT_NOK(backup_engine_->VerifyBackup(1, true));
|
|
ASSERT_NOK(backup_engine_->VerifyBackup(2, true));
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
// Test if BackupEngine will fail to create new backup if some table has been
|
|
// corrupted and the table file checksum is stored in the DB manifest
|
|
TEST_F(BackupableDBTest, TableFileCorruptedBeforeBackup) {
|
|
const int keys_iteration = 50000;
|
|
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
|
|
kNoShare);
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
// corrupt a random table file in the DB directory
|
|
ASSERT_OK(CorruptRandomTableFileInDB());
|
|
// file_checksum_gen_factory is null, and thus table checksum is not
|
|
// verified for creating a new backup; no correction is detected
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get()));
|
|
CloseDBAndBackupEngine();
|
|
// delete old files in db
|
|
ASSERT_OK(DestroyDB(dbname_, options_));
|
|
|
|
// Enable table file checksum in DB manifest
|
|
options_.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
|
|
kNoShare);
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
// corrupt a random table file in the DB directory
|
|
ASSERT_OK(CorruptRandomTableFileInDB());
|
|
// table file checksum is enabled so we should be able to detect any
|
|
// corruption
|
|
ASSERT_NOK(backup_engine_->CreateNewBackup(db_.get()));
|
|
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
// Test if BackupEngine will fail to create new backup if some table has been
|
|
// corrupted and the table file checksum is stored in the DB manifest for the
|
|
// case when backup table files will be stored in a shared directory
|
|
TEST_P(BackupableDBTestWithParam, TableFileCorruptedBeforeBackup) {
|
|
const int keys_iteration = 50000;
|
|
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */);
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
// corrupt a random table file in the DB directory
|
|
ASSERT_OK(CorruptRandomTableFileInDB());
|
|
// cannot detect corruption since DB manifest has no table checksums
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get()));
|
|
CloseDBAndBackupEngine();
|
|
// delete old files in db
|
|
ASSERT_OK(DestroyDB(dbname_, options_));
|
|
|
|
// Enable table checksums in DB manifest
|
|
options_.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */);
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
// corrupt a random table file in the DB directory
|
|
ASSERT_OK(CorruptRandomTableFileInDB());
|
|
// corruption is detected
|
|
ASSERT_NOK(backup_engine_->CreateNewBackup(db_.get()));
|
|
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, TableFileCorruptedDuringBackup) {
|
|
const int keys_iteration = 50000;
|
|
std::vector<std::shared_ptr<FileChecksumGenFactory>> fac{
|
|
nullptr, GetFileChecksumGenCrc32cFactory()};
|
|
for (auto& f : fac) {
|
|
options_.file_checksum_gen_factory = f;
|
|
if (f == nullptr) {
|
|
// When share_files_with_checksum is on, we calculate checksums of table
|
|
// files before and after copying. So we can test whether a corruption has
|
|
// happened during the file is copied to backup directory.
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
|
|
kShareWithChecksum);
|
|
} else {
|
|
// Default DB table file checksum is on, we calculate checksums of table
|
|
// files before copying to verify it with the one stored in DB manifest
|
|
// and also calculate checksum after copying. So we can test whether a
|
|
// corruption has happened during the file is copied to backup directory
|
|
// even if we do not place table files in shared_checksum directory.
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
|
|
kNoShare);
|
|
}
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
bool corrupted = false;
|
|
// corrupt files when copying to the backup directory
|
|
SyncPoint::GetInstance()->SetCallBack(
|
|
"BackupEngineImpl::CopyOrCreateFile:CorruptionDuringBackup",
|
|
[&](void* data) {
|
|
if (data != nullptr) {
|
|
Slice* d = reinterpret_cast<Slice*>(data);
|
|
if (!d->empty()) {
|
|
d->remove_suffix(1);
|
|
corrupted = true;
|
|
}
|
|
}
|
|
});
|
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
Status s = backup_engine_->CreateNewBackup(db_.get());
|
|
if (corrupted) {
|
|
ASSERT_NOK(s);
|
|
} else {
|
|
// should not in this path in normal cases
|
|
ASSERT_OK(s);
|
|
}
|
|
|
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
|
|
CloseDBAndBackupEngine();
|
|
// delete old files in db
|
|
ASSERT_OK(DestroyDB(dbname_, options_));
|
|
}
|
|
}
|
|
|
|
TEST_P(BackupableDBTestWithParam,
|
|
TableFileCorruptedDuringBackupWithDefaultDbChecksum) {
|
|
const int keys_iteration = 100000;
|
|
|
|
options_.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */);
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
bool corrupted = false;
|
|
// corrupt files when copying to the backup directory
|
|
SyncPoint::GetInstance()->SetCallBack(
|
|
"BackupEngineImpl::CopyOrCreateFile:CorruptionDuringBackup",
|
|
[&](void* data) {
|
|
if (data != nullptr) {
|
|
Slice* d = reinterpret_cast<Slice*>(data);
|
|
if (!d->empty()) {
|
|
d->remove_suffix(1);
|
|
corrupted = true;
|
|
}
|
|
}
|
|
});
|
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
Status s = backup_engine_->CreateNewBackup(db_.get());
|
|
if (corrupted) {
|
|
ASSERT_NOK(s);
|
|
} else {
|
|
// should not in this path in normal cases
|
|
ASSERT_OK(s);
|
|
}
|
|
|
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
|
|
CloseDBAndBackupEngine();
|
|
// delete old files in db
|
|
ASSERT_OK(DestroyDB(dbname_, options_));
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, InterruptCreationTest) {
|
|
// Interrupt backup creation by failing new writes and failing cleanup of the
|
|
// partial state. Then verify a subsequent backup can still succeed.
|
|
const int keys_iteration = 5000;
|
|
Random rnd(6);
|
|
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */);
|
|
FillDB(db_.get(), 0, keys_iteration);
|
|
test_backup_env_->SetLimitWrittenFiles(2);
|
|
test_backup_env_->SetDeleteFileFailure(true);
|
|
// should fail creation
|
|
ASSERT_FALSE(
|
|
backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)).ok());
|
|
CloseDBAndBackupEngine();
|
|
// should also fail cleanup so the tmp directory stays behind
|
|
ASSERT_OK(backup_chroot_env_->FileExists(backupdir_ + "/private/1/"));
|
|
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */);
|
|
test_backup_env_->SetLimitWrittenFiles(1000000);
|
|
test_backup_env_->SetDeleteFileFailure(false);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
|
|
// latest backup should have all the keys
|
|
CloseDBAndBackupEngine();
|
|
AssertBackupConsistency(0, 0, keys_iteration);
|
|
}
|
|
|
|
inline std::string OptionsPath(std::string ret, int backupID) {
|
|
ret += "/private/";
|
|
ret += std::to_string(backupID);
|
|
ret += "/";
|
|
return ret;
|
|
}
|
|
|
|
// Backup the LATEST options file to
|
|
// "<backup_dir>/private/<backup_id>/OPTIONS<number>"
|
|
|
|
TEST_F(BackupableDBTest, BackupOptions) {
|
|
OpenDBAndBackupEngine(true);
|
|
for (int i = 1; i < 5; i++) {
|
|
std::string name;
|
|
std::vector<std::string> filenames;
|
|
// Must reset() before reset(OpenDB()) again.
|
|
// Calling OpenDB() while *db_ is existing will cause LOCK issue
|
|
db_.reset();
|
|
db_.reset(OpenDB());
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
ROCKSDB_NAMESPACE::GetLatestOptionsFileName(db_->GetName(), options_.env,
|
|
&name);
|
|
ASSERT_OK(file_manager_->FileExists(OptionsPath(backupdir_, i) + name));
|
|
backup_chroot_env_->GetChildren(OptionsPath(backupdir_, i), &filenames);
|
|
for (auto fn : filenames) {
|
|
if (fn.compare(0, 7, "OPTIONS") == 0) {
|
|
ASSERT_EQ(name, fn);
|
|
}
|
|
}
|
|
}
|
|
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, SetOptionsBackupRaceCondition) {
|
|
OpenDBAndBackupEngine(true);
|
|
SyncPoint::GetInstance()->LoadDependency(
|
|
{{"CheckpointImpl::CreateCheckpoint:SavedLiveFiles1",
|
|
"BackupableDBTest::SetOptionsBackupRaceCondition:BeforeSetOptions"},
|
|
{"BackupableDBTest::SetOptionsBackupRaceCondition:AfterSetOptions",
|
|
"CheckpointImpl::CreateCheckpoint:SavedLiveFiles2"}});
|
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
ROCKSDB_NAMESPACE::port::Thread setoptions_thread{[this]() {
|
|
TEST_SYNC_POINT(
|
|
"BackupableDBTest::SetOptionsBackupRaceCondition:BeforeSetOptions");
|
|
DBImpl* dbi = static_cast<DBImpl*>(db_.get());
|
|
// Change arbitrary option to trigger OPTIONS file deletion
|
|
ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(),
|
|
{{"paranoid_file_checks", "false"}}));
|
|
ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(),
|
|
{{"paranoid_file_checks", "true"}}));
|
|
ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(),
|
|
{{"paranoid_file_checks", "false"}}));
|
|
TEST_SYNC_POINT(
|
|
"BackupableDBTest::SetOptionsBackupRaceCondition:AfterSetOptions");
|
|
}};
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get()));
|
|
setoptions_thread.join();
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
// This test verifies we don't delete the latest backup when read-only option is
|
|
// set
|
|
TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {
|
|
const int keys_iteration = 5000;
|
|
Random rnd(6);
|
|
Status s;
|
|
|
|
OpenDBAndBackupEngine(true);
|
|
// create five backups
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
ASSERT_OK(file_manager_->WriteToFile(backupdir_ + "/LATEST_BACKUP", "4"));
|
|
|
|
backupable_options_->destroy_old_data = false;
|
|
BackupEngineReadOnly* read_only_backup_engine;
|
|
ASSERT_OK(BackupEngineReadOnly::Open(backup_chroot_env_.get(),
|
|
*backupable_options_,
|
|
&read_only_backup_engine));
|
|
|
|
// assert that data from backup 5 is still here (even though LATEST_BACKUP
|
|
// says 4 is latest)
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/5"));
|
|
ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/5"));
|
|
|
|
// Behavior change: We now ignore LATEST_BACKUP contents. This means that
|
|
// we should have 5 backups, even if LATEST_BACKUP says 4.
|
|
std::vector<BackupInfo> backup_info;
|
|
read_only_backup_engine->GetBackupInfo(&backup_info);
|
|
ASSERT_EQ(5UL, backup_info.size());
|
|
delete read_only_backup_engine;
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, FailOverwritingBackups) {
|
|
options_.write_buffer_size = 1024 * 1024 * 1024; // 1GB
|
|
options_.disable_auto_compactions = true;
|
|
|
|
// create backups 1, 2, 3, 4, 5
|
|
OpenDBAndBackupEngine(true);
|
|
for (int i = 0; i < 5; ++i) {
|
|
CloseDBAndBackupEngine();
|
|
DeleteLogFiles();
|
|
OpenDBAndBackupEngine(false);
|
|
FillDB(db_.get(), 100 * i, 100 * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
// restore 3
|
|
OpenBackupEngine();
|
|
ASSERT_OK(backup_engine_->RestoreDBFromBackup(3, dbname_, dbname_));
|
|
CloseBackupEngine();
|
|
|
|
OpenDBAndBackupEngine(false);
|
|
FillDB(db_.get(), 0, 300);
|
|
Status s = backup_engine_->CreateNewBackup(db_.get(), true);
|
|
// the new backup fails because new table files
|
|
// clash with old table files from backups 4 and 5
|
|
// (since write_buffer_size is huge, we can be sure that
|
|
// each backup will generate only one sst file and that
|
|
// a file generated by a new backup is the same as
|
|
// sst file generated by backup 4)
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
ASSERT_OK(backup_engine_->DeleteBackup(4));
|
|
ASSERT_OK(backup_engine_->DeleteBackup(5));
|
|
// now, the backup can succeed
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
CloseDBAndBackupEngine();
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, NoShareTableFiles) {
|
|
const int keys_iteration = 5000;
|
|
OpenDBAndBackupEngine(true, false, kNoShare);
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(i % 2)));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 6);
|
|
}
|
|
}
|
|
|
|
// Verify that you can backup and restore with share_files_with_checksum on
|
|
TEST_F(BackupableDBTest, ShareTableFilesWithChecksums) {
|
|
const int keys_iteration = 5000;
|
|
OpenDBAndBackupEngine(true, false, kShareWithChecksum);
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(i % 2)));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 6);
|
|
}
|
|
}
|
|
|
|
// Verify that you can backup and restore using share_files_with_checksum set to
|
|
// false and then transition this option to true
|
|
TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsTransition) {
|
|
const int keys_iteration = 5000;
|
|
// set share_files_with_checksum to false
|
|
OpenDBAndBackupEngine(true, false, kShareNoChecksum);
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 6);
|
|
}
|
|
|
|
// set share_files_with_checksum to true and do some more backups
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false,
|
|
kShareWithChecksum);
|
|
for (int i = 5; i < 10; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify first (about to delete)
|
|
AssertBackupConsistency(1, 0, keys_iteration, keys_iteration * 11);
|
|
|
|
// For an extra challenge, make sure that GarbageCollect / DeleteBackup
|
|
// is OK even if we open without share_table_files
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false, kNoShare);
|
|
backup_engine_->DeleteBackup(1);
|
|
backup_engine_->GarbageCollect();
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify rest (not deleted)
|
|
for (int i = 1; i < 10; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 11);
|
|
}
|
|
}
|
|
|
|
// Verify backup and restore with share_files_with_checksum on and
|
|
// share_files_with_checksum_naming = kChecksumAndDbSessionId
|
|
TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsNewNaming) {
|
|
// Use session id in the name of SST file backup
|
|
ASSERT_TRUE(backupable_options_->share_files_with_checksum_naming ==
|
|
kChecksumAndDbSessionId);
|
|
const int keys_iteration = 5000;
|
|
OpenDBAndBackupEngine(true, false, kShareWithChecksum);
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(i % 2)));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 6);
|
|
}
|
|
}
|
|
|
|
// Verify backup and restore with share_files_with_checksum off and then
|
|
// transition this option and share_files_with_checksum_naming to be
|
|
// kChecksumAndDbSessionId
|
|
TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsNewNamingTransition) {
|
|
const int keys_iteration = 5000;
|
|
// We may set share_files_with_checksum_naming to kChecksumAndFileSize
|
|
// here but even if we don't, it should have no effect when
|
|
// share_files_with_checksum is false
|
|
ASSERT_TRUE(backupable_options_->share_files_with_checksum_naming ==
|
|
kChecksumAndDbSessionId);
|
|
// set share_files_with_checksum to false
|
|
OpenDBAndBackupEngine(true, false, kShareNoChecksum);
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 6);
|
|
}
|
|
|
|
// set share_files_with_checksum to true and do some more backups
|
|
// and use session id in the name of SST file backup
|
|
ASSERT_TRUE(backupable_options_->share_files_with_checksum_naming ==
|
|
kChecksumAndDbSessionId);
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false,
|
|
kShareWithChecksum);
|
|
for (int i = 5; i < 10; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify first (about to delete)
|
|
AssertBackupConsistency(1, 0, keys_iteration, keys_iteration * 11);
|
|
|
|
// For an extra challenge, make sure that GarbageCollect / DeleteBackup
|
|
// is OK even if we open without share_table_files but with
|
|
// share_files_with_checksum_naming being kChecksumAndDbSessionId
|
|
ASSERT_TRUE(backupable_options_->share_files_with_checksum_naming ==
|
|
kChecksumAndDbSessionId);
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false, kNoShare);
|
|
backup_engine_->DeleteBackup(1);
|
|
backup_engine_->GarbageCollect();
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify second (about to delete)
|
|
AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 11);
|
|
|
|
// Use checksum and file size for backup table file names and open without
|
|
// share_table_files
|
|
// Again, make sure that GarbageCollect / DeleteBackup is OK
|
|
backupable_options_->share_files_with_checksum_naming = kChecksumAndFileSize;
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false, kNoShare);
|
|
backup_engine_->DeleteBackup(2);
|
|
backup_engine_->GarbageCollect();
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify rest (not deleted)
|
|
for (int i = 1; i < 9; ++i) {
|
|
AssertBackupConsistency(i + 2, 0, keys_iteration * (i + 2),
|
|
keys_iteration * 11);
|
|
}
|
|
}
|
|
|
|
// Verify backup and restore with share_files_with_checksum on and transition
|
|
// from kChecksumAndFileSize to kChecksumAndDbSessionId
|
|
TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsNewNamingUpgrade) {
|
|
backupable_options_->share_files_with_checksum_naming = kChecksumAndFileSize;
|
|
const int keys_iteration = 5000;
|
|
// set share_files_with_checksum to true
|
|
OpenDBAndBackupEngine(true, false, kShareWithChecksum);
|
|
for (int i = 0; i < 5; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 6);
|
|
}
|
|
|
|
backupable_options_->share_files_with_checksum_naming =
|
|
kChecksumAndDbSessionId;
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false,
|
|
kShareWithChecksum);
|
|
for (int i = 5; i < 10; ++i) {
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify first (about to delete)
|
|
AssertBackupConsistency(1, 0, keys_iteration, keys_iteration * 11);
|
|
|
|
// For an extra challenge, make sure that GarbageCollect / DeleteBackup
|
|
// is OK even if we open without share_table_files
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false, kNoShare);
|
|
backup_engine_->DeleteBackup(1);
|
|
backup_engine_->GarbageCollect();
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify second (about to delete)
|
|
AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 11);
|
|
|
|
// Use checksum and file size for backup table file names and open without
|
|
// share_table_files
|
|
// Again, make sure that GarbageCollect / DeleteBackup is OK
|
|
backupable_options_->share_files_with_checksum_naming = kChecksumAndFileSize;
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false, kNoShare);
|
|
backup_engine_->DeleteBackup(2);
|
|
backup_engine_->GarbageCollect();
|
|
CloseDBAndBackupEngine();
|
|
|
|
// Verify rest (not deleted)
|
|
for (int i = 2; i < 10; ++i) {
|
|
AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
|
|
keys_iteration * 11);
|
|
}
|
|
}
|
|
|
|
// This test simulates cleaning up after aborted or incomplete creation
|
|
// of a new backup.
|
|
TEST_F(BackupableDBTest, DeleteTmpFiles) {
|
|
for (int cleanup_fn : {1, 2, 3, 4}) {
|
|
for (ShareOption shared_option : kAllShareOptions) {
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false /* dummy */,
|
|
shared_option);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get()));
|
|
BackupID next_id = 1;
|
|
BackupID oldest_id = std::numeric_limits<BackupID>::max();
|
|
{
|
|
std::vector<BackupInfo> backup_info;
|
|
backup_engine_->GetBackupInfo(&backup_info);
|
|
for (const auto& bi : backup_info) {
|
|
next_id = std::max(next_id, bi.backup_id + 1);
|
|
oldest_id = std::min(oldest_id, bi.backup_id);
|
|
}
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
// An aborted or incomplete new backup will always be in the next
|
|
// id (maybe more)
|
|
std::string next_private = "private/" + std::to_string(next_id);
|
|
|
|
// NOTE: both shared and shared_checksum should be cleaned up
|
|
// regardless of how the backup engine is opened.
|
|
std::vector<std::string> tmp_files_and_dirs;
|
|
for (const auto& dir_and_file : {
|
|
std::make_pair(std::string("shared"),
|
|
std::string(".00006.sst.tmp")),
|
|
std::make_pair(std::string("shared_checksum"),
|
|
std::string(".00007.sst.tmp")),
|
|
std::make_pair(next_private, std::string("00003.sst")),
|
|
}) {
|
|
std::string dir = backupdir_ + "/" + dir_and_file.first;
|
|
file_manager_->CreateDir(dir);
|
|
ASSERT_OK(file_manager_->FileExists(dir));
|
|
|
|
std::string file = dir + "/" + dir_and_file.second;
|
|
file_manager_->WriteToFile(file, "tmp");
|
|
ASSERT_OK(file_manager_->FileExists(file));
|
|
|
|
tmp_files_and_dirs.push_back(file);
|
|
}
|
|
if (cleanup_fn != /*CreateNewBackup*/ 4) {
|
|
// This exists after CreateNewBackup because it's deleted then
|
|
// re-created.
|
|
tmp_files_and_dirs.push_back(backupdir_ + "/" + next_private);
|
|
}
|
|
|
|
OpenDBAndBackupEngine(false /* destroy_old_data */, false /* dummy */,
|
|
shared_option);
|
|
// Need to call one of these explicitly to delete tmp files
|
|
switch (cleanup_fn) {
|
|
case 1:
|
|
ASSERT_OK(backup_engine_->GarbageCollect());
|
|
break;
|
|
case 2:
|
|
ASSERT_OK(backup_engine_->DeleteBackup(oldest_id));
|
|
break;
|
|
case 3:
|
|
ASSERT_OK(backup_engine_->PurgeOldBackups(1));
|
|
break;
|
|
case 4:
|
|
// Does a garbage collect if it sees that next private dir exists
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get()));
|
|
break;
|
|
default:
|
|
assert(false);
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
for (std::string file_or_dir : tmp_files_and_dirs) {
|
|
if (file_manager_->FileExists(file_or_dir) != Status::NotFound()) {
|
|
FAIL() << file_or_dir << " was expected to be deleted." << cleanup_fn;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, KeepLogFiles) {
|
|
backupable_options_->backup_log_files = false;
|
|
// basically infinite
|
|
options_.WAL_ttl_seconds = 24 * 60 * 60;
|
|
OpenDBAndBackupEngine(true);
|
|
FillDB(db_.get(), 0, 100);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
FillDB(db_.get(), 100, 200);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
|
FillDB(db_.get(), 200, 300);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
FillDB(db_.get(), 300, 400);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
FillDB(db_.get(), 400, 500);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
CloseDBAndBackupEngine();
|
|
|
|
// all data should be there if we call with keep_log_files = true
|
|
AssertBackupConsistency(0, 0, 500, 600, true);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, RateLimiting) {
|
|
size_t const kMicrosPerSec = 1000 * 1000LL;
|
|
uint64_t const MB = 1024 * 1024;
|
|
|
|
const std::vector<std::pair<uint64_t, uint64_t>> limits(
|
|
{{1 * MB, 5 * MB}, {2 * MB, 3 * MB}});
|
|
|
|
std::shared_ptr<RateLimiter> backupThrottler(NewGenericRateLimiter(1));
|
|
std::shared_ptr<RateLimiter> restoreThrottler(NewGenericRateLimiter(1));
|
|
|
|
for (bool makeThrottler : {false, true}) {
|
|
if (makeThrottler) {
|
|
backupable_options_->backup_rate_limiter = backupThrottler;
|
|
backupable_options_->restore_rate_limiter = restoreThrottler;
|
|
}
|
|
// iter 0 -- single threaded
|
|
// iter 1 -- multi threaded
|
|
for (int iter = 0; iter < 2; ++iter) {
|
|
for (const auto& limit : limits) {
|
|
// destroy old data
|
|
DestroyDB(dbname_, Options());
|
|
if (makeThrottler) {
|
|
backupThrottler->SetBytesPerSecond(limit.first);
|
|
restoreThrottler->SetBytesPerSecond(limit.second);
|
|
} else {
|
|
backupable_options_->backup_rate_limit = limit.first;
|
|
backupable_options_->restore_rate_limit = limit.second;
|
|
}
|
|
backupable_options_->max_background_operations = (iter == 0) ? 1 : 10;
|
|
options_.compression = kNoCompression;
|
|
OpenDBAndBackupEngine(true);
|
|
size_t bytes_written = FillDB(db_.get(), 0, 100000);
|
|
|
|
auto start_backup = db_chroot_env_->NowMicros();
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
|
auto backup_time = db_chroot_env_->NowMicros() - start_backup;
|
|
auto rate_limited_backup_time =
|
|
(bytes_written * kMicrosPerSec) / limit.first;
|
|
ASSERT_GT(backup_time, 0.8 * rate_limited_backup_time);
|
|
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenBackupEngine();
|
|
auto start_restore = db_chroot_env_->NowMicros();
|
|
ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_));
|
|
auto restore_time = db_chroot_env_->NowMicros() - start_restore;
|
|
CloseBackupEngine();
|
|
auto rate_limited_restore_time =
|
|
(bytes_written * kMicrosPerSec) / limit.second;
|
|
ASSERT_GT(restore_time, 0.8 * rate_limited_restore_time);
|
|
|
|
AssertBackupConsistency(0, 0, 100000, 100010);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
|
|
DestroyDB(dbname_, options_);
|
|
OpenDBAndBackupEngine(true);
|
|
FillDB(db_.get(), 0, 100);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
FillDB(db_.get(), 100, 200);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
|
|
backupable_options_->destroy_old_data = false;
|
|
test_backup_env_->ClearWrittenFiles();
|
|
test_backup_env_->SetLimitDeleteFiles(0);
|
|
BackupEngineReadOnly* read_only_backup_engine;
|
|
ASSERT_OK(BackupEngineReadOnly::Open(
|
|
db_chroot_env_.get(), *backupable_options_, &read_only_backup_engine));
|
|
std::vector<BackupInfo> backup_info;
|
|
read_only_backup_engine->GetBackupInfo(&backup_info);
|
|
ASSERT_EQ(backup_info.size(), 2U);
|
|
|
|
RestoreOptions restore_options(false);
|
|
ASSERT_OK(read_only_backup_engine->RestoreDBFromLatestBackup(
|
|
dbname_, dbname_, restore_options));
|
|
delete read_only_backup_engine;
|
|
std::vector<std::string> should_have_written;
|
|
test_backup_env_->AssertWrittenFiles(should_have_written);
|
|
|
|
DB* db = OpenDB();
|
|
AssertExists(db, 0, 200);
|
|
delete db;
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
|
|
DestroyDB(dbname_, options_);
|
|
OpenDBAndBackupEngine(true);
|
|
FillDB(db_.get(), 0, 100);
|
|
bool is_callback_invoked = false;
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(
|
|
db_.get(), true,
|
|
[&is_callback_invoked]() { is_callback_invoked = true; }));
|
|
|
|
ASSERT_TRUE(is_callback_invoked);
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
|
|
DestroyDB(dbname_, options_);
|
|
OpenDBAndBackupEngine(true);
|
|
|
|
backup_chroot_env_->CreateDirIfMissing(backupdir_ + "/shared");
|
|
std::string file_five = backupdir_ + "/shared/000007.sst";
|
|
std::string file_five_contents = "I'm not really a sst file";
|
|
// this depends on the fact that 00007.sst is the first file created by the DB
|
|
ASSERT_OK(file_manager_->WriteToFile(file_five, file_five_contents));
|
|
|
|
FillDB(db_.get(), 0, 100);
|
|
// backup overwrites file 000007.sst
|
|
ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
|
|
|
|
std::string new_file_five_contents;
|
|
ASSERT_OK(ReadFileToString(backup_chroot_env_.get(), file_five,
|
|
&new_file_five_contents));
|
|
// file 000007.sst was overwritten
|
|
ASSERT_TRUE(new_file_five_contents != file_five_contents);
|
|
|
|
CloseDBAndBackupEngine();
|
|
|
|
AssertBackupConsistency(0, 0, 100);
|
|
}
|
|
|
|
// Test that we properly propagate Env failures
|
|
TEST_F(BackupableDBTest, EnvFailures) {
|
|
BackupEngine* backup_engine;
|
|
|
|
// get children failure
|
|
{
|
|
test_backup_env_->SetGetChildrenFailure(true);
|
|
ASSERT_NOK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
test_backup_env_->SetGetChildrenFailure(false);
|
|
}
|
|
|
|
// created dir failure
|
|
{
|
|
test_backup_env_->SetCreateDirIfMissingFailure(true);
|
|
ASSERT_NOK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
test_backup_env_->SetCreateDirIfMissingFailure(false);
|
|
}
|
|
|
|
// new directory failure
|
|
{
|
|
test_backup_env_->SetNewDirectoryFailure(true);
|
|
ASSERT_NOK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
test_backup_env_->SetNewDirectoryFailure(false);
|
|
}
|
|
|
|
// Read from meta-file failure
|
|
{
|
|
DestroyDB(dbname_, options_);
|
|
OpenDBAndBackupEngine(true);
|
|
FillDB(db_.get(), 0, 100);
|
|
ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
|
|
CloseDBAndBackupEngine();
|
|
test_backup_env_->SetDummySequentialFile(true);
|
|
test_backup_env_->SetDummySequentialFileFailReads(true);
|
|
backupable_options_->destroy_old_data = false;
|
|
ASSERT_NOK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
test_backup_env_->SetDummySequentialFile(false);
|
|
test_backup_env_->SetDummySequentialFileFailReads(false);
|
|
}
|
|
|
|
// no failure
|
|
{
|
|
ASSERT_OK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
delete backup_engine;
|
|
}
|
|
}
|
|
|
|
// Verify manifest can roll while a backup is being created with the old
|
|
// manifest.
|
|
TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
|
|
DestroyDB(dbname_, options_);
|
|
options_.max_manifest_file_size = 0; // always rollover manifest for file add
|
|
OpenDBAndBackupEngine(true);
|
|
FillDB(db_.get(), 0, 100);
|
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
|
|
{"CheckpointImpl::CreateCheckpoint:SavedLiveFiles1",
|
|
"VersionSet::LogAndApply:WriteManifest"},
|
|
{"VersionSet::LogAndApply:WriteManifestDone",
|
|
"CheckpointImpl::CreateCheckpoint:SavedLiveFiles2"},
|
|
});
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
|
|
ROCKSDB_NAMESPACE::port::Thread flush_thread{
|
|
[this]() { ASSERT_OK(db_->Flush(FlushOptions())); }};
|
|
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
|
|
|
flush_thread.join();
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
|
|
// The last manifest roll would've already been cleaned up by the full scan
|
|
// that happens when CreateNewBackup invokes EnableFileDeletions. We need to
|
|
// trigger another roll to verify non-full scan purges stale manifests.
|
|
DBImpl* db_impl = static_cast_with_check<DBImpl>(db_.get());
|
|
std::string prev_manifest_path =
|
|
DescriptorFileName(dbname_, db_impl->TEST_Current_Manifest_FileNo());
|
|
FillDB(db_.get(), 0, 100);
|
|
ASSERT_OK(db_chroot_env_->FileExists(prev_manifest_path));
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
ASSERT_TRUE(db_chroot_env_->FileExists(prev_manifest_path).IsNotFound());
|
|
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
AssertBackupConsistency(0, 0, 100);
|
|
}
|
|
|
|
// see https://github.com/facebook/rocksdb/issues/921
|
|
TEST_F(BackupableDBTest, Issue921Test) {
|
|
BackupEngine* backup_engine;
|
|
backupable_options_->share_table_files = false;
|
|
backup_chroot_env_->CreateDirIfMissing(backupable_options_->backup_dir);
|
|
backupable_options_->backup_dir += "/new_dir";
|
|
ASSERT_OK(BackupEngine::Open(backup_chroot_env_.get(), *backupable_options_,
|
|
&backup_engine));
|
|
|
|
delete backup_engine;
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, BackupWithMetadata) {
|
|
const int keys_iteration = 5000;
|
|
OpenDBAndBackupEngine(true);
|
|
// create five backups
|
|
for (int i = 0; i < 5; ++i) {
|
|
const std::string metadata = std::to_string(i);
|
|
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
|
|
ASSERT_OK(
|
|
backup_engine_->CreateNewBackupWithMetadata(db_.get(), metadata, true));
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenDBAndBackupEngine();
|
|
std::vector<BackupInfo> backup_infos;
|
|
backup_engine_->GetBackupInfo(&backup_infos);
|
|
ASSERT_EQ(5, backup_infos.size());
|
|
for (int i = 0; i < 5; i++) {
|
|
ASSERT_EQ(std::to_string(i), backup_infos[i].app_metadata);
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, BinaryMetadata) {
|
|
OpenDBAndBackupEngine(true);
|
|
std::string binaryMetadata = "abc\ndef";
|
|
binaryMetadata.push_back('\0');
|
|
binaryMetadata.append("ghi");
|
|
ASSERT_OK(
|
|
backup_engine_->CreateNewBackupWithMetadata(db_.get(), binaryMetadata));
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenDBAndBackupEngine();
|
|
std::vector<BackupInfo> backup_infos;
|
|
backup_engine_->GetBackupInfo(&backup_infos);
|
|
ASSERT_EQ(1, backup_infos.size());
|
|
ASSERT_EQ(binaryMetadata, backup_infos[0].app_metadata);
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, MetadataTooLarge) {
|
|
OpenDBAndBackupEngine(true);
|
|
std::string largeMetadata(1024 * 1024 + 1, 0);
|
|
ASSERT_NOK(
|
|
backup_engine_->CreateNewBackupWithMetadata(db_.get(), largeMetadata));
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, LimitBackupsOpened) {
|
|
// Verify the specified max backups are opened, including skipping over
|
|
// corrupted backups.
|
|
//
|
|
// Setup:
|
|
// - backups 1, 2, and 4 are valid
|
|
// - backup 3 is corrupt
|
|
// - max_valid_backups_to_open == 2
|
|
//
|
|
// Expectation: the engine opens backups 4 and 2 since those are latest two
|
|
// non-corrupt backups.
|
|
const int kNumKeys = 5000;
|
|
OpenDBAndBackupEngine(true);
|
|
for (int i = 1; i <= 4; ++i) {
|
|
FillDB(db_.get(), kNumKeys * i, kNumKeys * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
if (i == 3) {
|
|
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/3", 3));
|
|
}
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
backupable_options_->max_valid_backups_to_open = 2;
|
|
backupable_options_->destroy_old_data = false;
|
|
BackupEngineReadOnly* read_only_backup_engine;
|
|
ASSERT_OK(BackupEngineReadOnly::Open(backup_chroot_env_.get(),
|
|
*backupable_options_,
|
|
&read_only_backup_engine));
|
|
|
|
std::vector<BackupInfo> backup_infos;
|
|
read_only_backup_engine->GetBackupInfo(&backup_infos);
|
|
ASSERT_EQ(2, backup_infos.size());
|
|
ASSERT_EQ(2, backup_infos[0].backup_id);
|
|
ASSERT_EQ(4, backup_infos[1].backup_id);
|
|
delete read_only_backup_engine;
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, IgnoreLimitBackupsOpenedWhenNotReadOnly) {
|
|
// Verify the specified max_valid_backups_to_open is ignored if the engine
|
|
// is not read-only.
|
|
//
|
|
// Setup:
|
|
// - backups 1, 2, and 4 are valid
|
|
// - backup 3 is corrupt
|
|
// - max_valid_backups_to_open == 2
|
|
//
|
|
// Expectation: the engine opens backups 4, 2, and 1 since those are latest
|
|
// non-corrupt backups, by ignoring max_valid_backups_to_open == 2.
|
|
const int kNumKeys = 5000;
|
|
OpenDBAndBackupEngine(true);
|
|
for (int i = 1; i <= 4; ++i) {
|
|
FillDB(db_.get(), kNumKeys * i, kNumKeys * (i + 1));
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
if (i == 3) {
|
|
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/3", 3));
|
|
}
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
backupable_options_->max_valid_backups_to_open = 2;
|
|
OpenDBAndBackupEngine();
|
|
std::vector<BackupInfo> backup_infos;
|
|
backup_engine_->GetBackupInfo(&backup_infos);
|
|
ASSERT_EQ(3, backup_infos.size());
|
|
ASSERT_EQ(1, backup_infos[0].backup_id);
|
|
ASSERT_EQ(2, backup_infos[1].backup_id);
|
|
ASSERT_EQ(4, backup_infos[2].backup_id);
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, CreateWhenLatestBackupCorrupted) {
|
|
// we should pick an ID greater than corrupted backups' IDs so creation can
|
|
// succeed even when latest backup is corrupted.
|
|
const int kNumKeys = 5000;
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */);
|
|
FillDB(db_.get(), 0 /* from */, kNumKeys);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(),
|
|
true /* flush_before_backup */));
|
|
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/1",
|
|
3 /* bytes_to_corrupt */));
|
|
CloseDBAndBackupEngine();
|
|
|
|
OpenDBAndBackupEngine();
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(),
|
|
true /* flush_before_backup */));
|
|
std::vector<BackupInfo> backup_infos;
|
|
backup_engine_->GetBackupInfo(&backup_infos);
|
|
ASSERT_EQ(1, backup_infos.size());
|
|
ASSERT_EQ(2, backup_infos[0].backup_id);
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, WriteOnlyEngineNoSharedFileDeletion) {
|
|
// Verifies a write-only BackupEngine does not delete files belonging to valid
|
|
// backups when GarbageCollect, PurgeOldBackups, or DeleteBackup are called.
|
|
const int kNumKeys = 5000;
|
|
for (int i = 0; i < 3; ++i) {
|
|
OpenDBAndBackupEngine(i == 0 /* destroy_old_data */);
|
|
FillDB(db_.get(), i * kNumKeys, (i + 1) * kNumKeys);
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
|
|
CloseDBAndBackupEngine();
|
|
|
|
backupable_options_->max_valid_backups_to_open = 0;
|
|
OpenDBAndBackupEngine();
|
|
switch (i) {
|
|
case 0:
|
|
ASSERT_OK(backup_engine_->GarbageCollect());
|
|
break;
|
|
case 1:
|
|
ASSERT_OK(backup_engine_->PurgeOldBackups(1 /* num_backups_to_keep */));
|
|
break;
|
|
case 2:
|
|
ASSERT_OK(backup_engine_->DeleteBackup(2 /* backup_id */));
|
|
break;
|
|
default:
|
|
assert(false);
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
backupable_options_->max_valid_backups_to_open = port::kMaxInt32;
|
|
AssertBackupConsistency(i + 1, 0, (i + 1) * kNumKeys);
|
|
}
|
|
}
|
|
|
|
TEST_P(BackupableDBTestWithParam, BackupUsingDirectIO) {
|
|
// Tests direct I/O on the backup engine's reads and writes on the DB env and
|
|
// backup env
|
|
// We use ChrootEnv underneath so the below line checks for direct I/O support
|
|
// in the chroot directory, not the true filesystem root.
|
|
if (!test::IsDirectIOSupported(test_db_env_.get(), "/")) {
|
|
return;
|
|
}
|
|
const int kNumKeysPerBackup = 100;
|
|
const int kNumBackups = 3;
|
|
options_.use_direct_reads = true;
|
|
OpenDBAndBackupEngine(true /* destroy_old_data */);
|
|
for (int i = 0; i < kNumBackups; ++i) {
|
|
FillDB(db_.get(), i * kNumKeysPerBackup /* from */,
|
|
(i + 1) * kNumKeysPerBackup /* to */);
|
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
|
|
// Clear the file open counters and then do a bunch of backup engine ops.
|
|
// For all ops, files should be opened in direct mode.
|
|
test_backup_env_->ClearFileOpenCounters();
|
|
test_db_env_->ClearFileOpenCounters();
|
|
CloseBackupEngine();
|
|
OpenBackupEngine();
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(),
|
|
false /* flush_before_backup */));
|
|
ASSERT_OK(backup_engine_->VerifyBackup(i + 1));
|
|
CloseBackupEngine();
|
|
OpenBackupEngine();
|
|
std::vector<BackupInfo> backup_infos;
|
|
backup_engine_->GetBackupInfo(&backup_infos);
|
|
ASSERT_EQ(static_cast<size_t>(i + 1), backup_infos.size());
|
|
|
|
// Verify backup engine always opened files with direct I/O
|
|
ASSERT_EQ(0, test_db_env_->num_writers());
|
|
ASSERT_EQ(0, test_db_env_->num_rand_readers());
|
|
ASSERT_GT(test_db_env_->num_direct_seq_readers(), 0);
|
|
// Currently the DB doesn't support reading WALs or manifest with direct
|
|
// I/O, so subtract two.
|
|
ASSERT_EQ(test_db_env_->num_seq_readers() - 2,
|
|
test_db_env_->num_direct_seq_readers());
|
|
ASSERT_EQ(0, test_db_env_->num_rand_readers());
|
|
}
|
|
CloseDBAndBackupEngine();
|
|
|
|
for (int i = 0; i < kNumBackups; ++i) {
|
|
AssertBackupConsistency(i + 1 /* backup_id */,
|
|
i * kNumKeysPerBackup /* start_exist */,
|
|
(i + 1) * kNumKeysPerBackup /* end_exist */,
|
|
(i + 2) * kNumKeysPerBackup /* end */);
|
|
}
|
|
}
|
|
|
|
TEST_F(BackupableDBTest, BackgroundThreadCpuPriority) {
|
|
std::atomic<CpuPriority> priority(CpuPriority::kNormal);
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
"BackupEngineImpl::Initialize:SetCpuPriority", [&](void* new_priority) {
|
|
priority.store(*reinterpret_cast<CpuPriority*>(new_priority));
|
|
});
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
|
|
// 1 thread is easier to test, otherwise, we may not be sure which thread
|
|
// actually does the work during CreateNewBackup.
|
|
backupable_options_->max_background_operations = 1;
|
|
OpenDBAndBackupEngine(true);
|
|
|
|
{
|
|
FillDB(db_.get(), 0, 100);
|
|
|
|
// by default, cpu priority is not changed.
|
|
CreateBackupOptions options;
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(options, db_.get()));
|
|
|
|
ASSERT_EQ(priority, CpuPriority::kNormal);
|
|
}
|
|
|
|
{
|
|
FillDB(db_.get(), 101, 200);
|
|
|
|
// decrease cpu priority from normal to low.
|
|
CreateBackupOptions options;
|
|
options.decrease_background_thread_cpu_priority = true;
|
|
options.background_thread_cpu_priority = CpuPriority::kLow;
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(options, db_.get()));
|
|
|
|
ASSERT_EQ(priority, CpuPriority::kLow);
|
|
}
|
|
|
|
{
|
|
FillDB(db_.get(), 201, 300);
|
|
|
|
// try to upgrade cpu priority back to normal,
|
|
// the priority should still low.
|
|
CreateBackupOptions options;
|
|
options.decrease_background_thread_cpu_priority = true;
|
|
options.background_thread_cpu_priority = CpuPriority::kNormal;
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(options, db_.get()));
|
|
|
|
ASSERT_EQ(priority, CpuPriority::kLow);
|
|
}
|
|
|
|
{
|
|
FillDB(db_.get(), 301, 400);
|
|
|
|
// decrease cpu priority from low to idle.
|
|
CreateBackupOptions options;
|
|
options.decrease_background_thread_cpu_priority = true;
|
|
options.background_thread_cpu_priority = CpuPriority::kIdle;
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(options, db_.get()));
|
|
|
|
ASSERT_EQ(priority, CpuPriority::kIdle);
|
|
}
|
|
|
|
{
|
|
FillDB(db_.get(), 301, 400);
|
|
|
|
// reset priority to later verify that it's not updated by SetCpuPriority.
|
|
priority = CpuPriority::kNormal;
|
|
|
|
// setting the same cpu priority won't call SetCpuPriority.
|
|
CreateBackupOptions options;
|
|
options.decrease_background_thread_cpu_priority = true;
|
|
options.background_thread_cpu_priority = CpuPriority::kIdle;
|
|
ASSERT_OK(backup_engine_->CreateNewBackup(options, db_.get()));
|
|
|
|
ASSERT_EQ(priority, CpuPriority::kNormal);
|
|
}
|
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
CloseDBAndBackupEngine();
|
|
DestroyDB(dbname_, options_);
|
|
}
|
|
|
|
} // anon namespace
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
|
|
int main(int argc, char** argv) {
|
|
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|
|
|
|
#else
|
|
#include <stdio.h>
|
|
|
|
int main(int /*argc*/, char** /*argv*/) {
|
|
fprintf(stderr, "SKIPPED as BackupableDB is not supported in ROCKSDB_LITE\n");
|
|
return 0;
|
|
}
|
|
|
|
#endif // !defined(ROCKSDB_LITE) && !defined(OS_WIN)
|