dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-26 07:30:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Isolate db env and backup Env in unit tests

Browse source 
Summary:
- Used ChrootEnv so the database and backup Envs are isolated in the filesystem.
- Removed DifferentEnvs test since now every test uses different Envs

Depends on D57543

Test Plan:
- ran backupable_db_test
- verified backupable_db_test now catches the bug when D57159 is backed out (this bug previously passed through the test cases, which motivated this change)

Reviewers: sdong, lightmark, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57615
This commit is contained in:
Andrew Kryczka 2016-05-11 08:18:44 -07:00
parent 560358dc93
commit e61ba052b3
1 changed files with 56 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

142
utilities/backupable/backupable_db_test.cc
View file

@ -9,25 +9,24 @@
#ifndef ROCKSDB_LITE
#include <string>
#include <algorithm>
#include <iostream>
#include <string>
#include "db/db_impl.h"
#include "db/filename.h"
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/types.h"
#include "rocksdb/transaction_log.h"
#include "rocksdb/types.h"
#include "rocksdb/utilities/backupable_db.h"
#include "util/env_chroot.h"
#include "util/file_reader_writer.h"
#include "util/testharness.h"
#include "util/random.h"
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/string_util.h"
#include "util/sync_point.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "util/mock_env.h"
namespace rocksdb {
@ -428,15 +427,19 @@ class BackupableDBTest : public testing::Test {
public:
BackupableDBTest() {
// set up files
dbname_ = test::TmpDir() + "/backupable_db";
backupdir_ = test::TmpDir() + "/backupable_db_backup";
std::string db_chroot = test::TmpDir() + "/backupable_db";
std::string backup_chroot = test::TmpDir() + "/backupable_db_backup";
Env::Default()->CreateDir(db_chroot);
Env::Default()->CreateDir(backup_chroot);
dbname_ = "/tempdb";
backupdir_ = "/tempbk";
// set up envs
env_ = Env::Default();
mock_env_.reset(new MockEnv(env_));
test_db_env_.reset(new TestEnv(env_));
test_backup_env_.reset(new TestEnv(env_));
file_manager_.reset(new FileManager(env_));
db_chroot_env_.reset(NewChrootEnv(Env::Default(), db_chroot));
backup_chroot_env_.reset(NewChrootEnv(Env::Default(), backup_chroot));
test_db_env_.reset(new TestEnv(db_chroot_env_.get()));
test_backup_env_.reset(new TestEnv(backup_chroot_env_.get()));
file_manager_.reset(new FileManager(backup_chroot_env_.get()));
// set up db options
options_.create_if_missing = true;
@ -447,8 +450,7 @@ class BackupableDBTest : public testing::Test {
// Create logger
DBOptions logger_options;
logger_options.env = env_;
logger_options.db_log_dir = backupdir_;
logger_options.env = db_chroot_env_.get();
CreateLoggerFromOptions(dbname_, logger_options, &logger_);
// set up backup db options
@ -459,7 +461,7 @@ class BackupableDBTest : public testing::Test {
backupable_options_->max_background_operations = 7;
// delete old files in db
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
DB* OpenDB() {
@ -548,13 +550,13 @@ class BackupableDBTest : public testing::Test {
void DeleteLogFiles() {
std::vector<std::string> delete_logs;
env_->GetChildren(dbname_, &delete_logs);
db_chroot_env_->GetChildren(dbname_, &delete_logs);
for (auto f : delete_logs) {
uint64_t number;
FileType type;
bool ok = ParseFileName(f, &number, &type);
if (ok && type == kLogFile) {
env_->DeleteFile(dbname_ + "/" + f);
db_chroot_env_->DeleteFile(dbname_ + "/" + f);
}
}
}
@ -568,8 +570,8 @@ class BackupableDBTest : public testing::Test {
std::shared_ptr<Logger> logger_;
// envs
Env* env_;
unique_ptr<MockEnv> mock_env_;
unique_ptr<Env> db_chroot_env_;
unique_ptr<Env> backup_chroot_env_;
unique_ptr<TestEnv> test_db_env_;
unique_ptr<TestEnv> test_backup_env_;
unique_ptr<FileManager> file_manager_;
@ -642,7 +644,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
// second iter -- don't flush before backup
for (int iter = 0; iter < 2; ++iter) {
// delete old data
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
bool destroy_data = true;
// every iteration --
@ -659,7 +661,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
FillDB(db_.get(), keys_iteration * i, fill_up_to);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
// ---- make sure it's empty ----
DB* db = OpenDB();
@ -687,7 +689,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
const int max_key = keys_iteration * 4 + 10;
Random rnd(7);
// delete old data
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
// write some data, backup, repeat
@ -706,7 +708,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
}
// close and destroy
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
// ---- make sure it's empty ----
DB* db = OpenDB();
@ -763,7 +765,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
"/private/1.tmp/CURRENT", "/private/1.tmp/MANIFEST-01",
"/private/1.tmp/00011.log", "/meta/1.tmp",
"/LATEST_BACKUP.tmp"};
AppendPath(dbname_ + "_backup", should_have_written);
AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written);
// should write 4 new DB files + LATEST_BACKUP + one meta file
@ -784,7 +786,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
"/meta/2.tmp",
"/LATEST_BACKUP.tmp"
};
AppendPath(dbname_ + "_backup", should_have_written);
AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written);
ASSERT_OK(backup_engine_->DeleteBackup(1));
@ -805,41 +807,6 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
CloseDBAndBackupEngine();
}
// Verify that backup works when the database environment is not the same as
// the backup environment
// TODO(agf): Make all/most tests use different db and backup environments.
// This will probably require more implementation of MockEnv.
// For example, MockEnv::RenameFile() must be able to rename
// directories.
TEST_F(BackupableDBTest, DifferentEnvs) {
test_db_env_.reset(new TestEnv(mock_env_.get()));
options_.env = test_db_env_.get();
OpenDBAndBackupEngine(true, true);
// should write 5 DB files + LATEST_BACKUP + one meta file
test_backup_env_->SetLimitWrittenFiles(7);
test_backup_env_->ClearWrittenFiles();
test_db_env_->SetLimitWrittenFiles(0);
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
"/CURRENT", "/MANIFEST-01" };
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
CloseDBAndBackupEngine();
// try simple backup and verify correctness
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
test_db_env_->SetFilenamesForMockedAttrs({});
AssertBackupConsistency(0, 0, 100, 500);
}
// test various kind of corruptions that may happen:
// 1. Not able to write a file for backup - that backup should fail,
// everything else should work
@ -893,7 +860,7 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
// assert that we wrote 6 to LATEST_BACKUP
{
std::string latest_backup_contents;
ReadFileToString(env_, backupdir_ + "/LATEST_BACKUP",
ReadFileToString(backup_chroot_env_.get(), backupdir_ + "/LATEST_BACKUP",
&latest_backup_contents);
ASSERT_EQ(std::atol(latest_backup_contents.c_str()), 6);
}
@ -993,7 +960,8 @@ TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {
backupable_options_->destroy_old_data = false;
BackupEngineReadOnly* read_only_backup_engine;
ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_,
ASSERT_OK(BackupEngineReadOnly::Open(backup_chroot_env_.get(),
*backupable_options_,
&read_only_backup_engine));
// assert that data from backup 5 is still here (even though LATEST_BACKUP
@ -1160,7 +1128,7 @@ TEST_F(BackupableDBTest, RateLimiting) {
for (const auto& limit : limits) {
// destroy old data
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
backupable_options_->backup_rate_limit = limit.first;
backupable_options_->restore_rate_limit = limit.second;
@ -1169,9 +1137,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
OpenDBAndBackupEngine(true);
size_t bytes_written = FillDB(db_.get(), 0, 100000);
auto start_backup = env_->NowMicros();
auto start_backup = db_chroot_env_->NowMicros();
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
auto backup_time = env_->NowMicros() - start_backup;
auto backup_time = db_chroot_env_->NowMicros() - start_backup;
auto rate_limited_backup_time = (bytes_written * kMicrosPerSec) /
backupable_options_->backup_rate_limit;
ASSERT_GT(backup_time, 0.8 * rate_limited_backup_time);
@ -1179,9 +1147,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
CloseDBAndBackupEngine();
OpenBackupEngine();
auto start_restore = env_->NowMicros();
auto start_restore = db_chroot_env_->NowMicros();
ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_));
auto restore_time = env_->NowMicros() - start_restore;
auto restore_time = db_chroot_env_->NowMicros() - start_restore;
CloseBackupEngine();
auto rate_limited_restore_time = (bytes_written * kMicrosPerSec) /
backupable_options_->restore_rate_limit;
@ -1193,21 +1161,21 @@ TEST_F(BackupableDBTest, RateLimiting) {
}
TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
FillDB(db_.get(), 100, 200);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
backupable_options_->destroy_old_data = false;
test_backup_env_->ClearWrittenFiles();
test_backup_env_->SetLimitDeleteFiles(0);
BackupEngineReadOnly* read_only_backup_engine;
ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_,
&read_only_backup_engine));
ASSERT_OK(BackupEngineReadOnly::Open(
db_chroot_env_.get(), *backupable_options_, &read_only_backup_engine));
std::vector<BackupInfo> backup_info;
read_only_backup_engine->GetBackupInfo(&backup_info);
ASSERT_EQ(backup_info.size(), 2U);
@ -1225,7 +1193,7 @@ TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
}
TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
bool is_callback_invoked = false;
@ -1235,14 +1203,14 @@ TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
ASSERT_TRUE(is_callback_invoked);
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
env_->CreateDirIfMissing(backupdir_ + "/shared");
backup_chroot_env_->CreateDirIfMissing(backupdir_ + "/shared");
std::string file_five = backupdir_ + "/shared/000007.sst";
std::string file_five_contents = "I'm not really a sst file";
// this depends on the fact that 00007.sst is the first file created by the DB
@ -1253,7 +1221,8 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
std::string new_file_five_contents;
ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents));
ASSERT_OK(ReadFileToString(backup_chroot_env_.get(), file_five,
&new_file_five_contents));
// file 000007.sst was overwritten
ASSERT_TRUE(new_file_five_contents != file_five_contents);
@ -1301,7 +1270,7 @@ TEST_F(BackupableDBTest, EnvFailures) {
// Verify manifest can roll while a backup is being created with the old
// manifest.
TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
options_.max_manifest_file_size = 0; // always rollover manifest for file add
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
@ -1328,12 +1297,12 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
std::string prev_manifest_path =
DescriptorFileName(dbname_, db_impl->TEST_Current_Manifest_FileNo());
FillDB(db_.get(), 0, 100);
ASSERT_OK(env_->FileExists(prev_manifest_path));
ASSERT_OK(db_chroot_env_->FileExists(prev_manifest_path));
ASSERT_OK(db_->Flush(FlushOptions()));
ASSERT_TRUE(env_->FileExists(prev_manifest_path).IsNotFound());
ASSERT_TRUE(db_chroot_env_->FileExists(prev_manifest_path).IsNotFound());
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
AssertBackupConsistency(0, 0, 100);
}
@ -1341,9 +1310,10 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
TEST_F(BackupableDBTest, Issue921Test) {
BackupEngine* backup_engine;
backupable_options_->share_table_files = false;
env_->CreateDirIfMissing(backupable_options_->backup_dir);
backup_chroot_env_->CreateDirIfMissing(backupable_options_->backup_dir);
backupable_options_->backup_dir += "/new_dir";
ASSERT_OK(BackupEngine::Open(env_, *backupable_options_, &backup_engine));
ASSERT_OK(BackupEngine::Open(backup_chroot_env_.get(), *backupable_options_,
&backup_engine));
delete backup_engine;
}
@ -1368,7 +1338,7 @@ TEST_F(BackupableDBTest, BackupWithMetadata) {
ASSERT_EQ(std::to_string(i), backup_infos[i].app_metadata);
}
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
TEST_F(BackupableDBTest, BinaryMetadata) {
@ -1386,7 +1356,7 @@ TEST_F(BackupableDBTest, BinaryMetadata) {
ASSERT_EQ(1, backup_infos.size());
ASSERT_EQ(binaryMetadata, backup_infos[0].app_metadata);
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
TEST_F(BackupableDBTest, MetadataTooLarge) {
@ -1395,7 +1365,7 @@ TEST_F(BackupableDBTest, MetadataTooLarge) {
ASSERT_NOK(
backup_engine_->CreateNewBackupWithMetadata(db_.get(), largeMetadata));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
} // anon namespace