dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-29 18:33:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Isolate db env and backup Env in unit tests

Browse source 
Summary:
- Used ChrootEnv so the database and backup Envs are isolated in the filesystem.
- Removed DifferentEnvs test since now every test uses different Envs

Depends on D57543

Test Plan:
- ran backupable_db_test
- verified backupable_db_test now catches the bug when D57159 is backed out (this bug previously passed through the test cases, which motivated this change)

Reviewers: sdong, lightmark, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57615
This commit is contained in:
Andrew Kryczka 2016-05-11 08:18:44 -07:00
parent 560358dc93
commit e61ba052b3
1 changed files with 56 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

142
utilities/backupable/backupable_db_test.cc
View file

@ -9,25 +9,24 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include <string>
#include <algorithm> #include <algorithm>
#include <iostream> #include <string>
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/filename.h" #include "db/filename.h"
#include "port/port.h" #include "port/port.h"
#include "port/stack_trace.h" #include "port/stack_trace.h"
#include "rocksdb/types.h"
#include "rocksdb/transaction_log.h" #include "rocksdb/transaction_log.h"
#include "rocksdb/types.h"
#include "rocksdb/utilities/backupable_db.h" #include "rocksdb/utilities/backupable_db.h"
#include "util/env_chroot.h"
#include "util/file_reader_writer.h" #include "util/file_reader_writer.h"
#include "util/testharness.h"
#include "util/random.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
#include "util/random.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "util/sync_point.h" #include "util/sync_point.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
#include "util/mock_env.h"
namespace rocksdb { namespace rocksdb {
@ -428,15 +427,19 @@ class BackupableDBTest : public testing::Test {
public: public:
BackupableDBTest() { BackupableDBTest() {
// set up files // set up files
dbname_ = test::TmpDir() + "/backupable_db"; std::string db_chroot = test::TmpDir() + "/backupable_db";
backupdir_ = test::TmpDir() + "/backupable_db_backup"; std::string backup_chroot = test::TmpDir() + "/backupable_db_backup";
Env::Default()->CreateDir(db_chroot);
Env::Default()->CreateDir(backup_chroot);
dbname_ = "/tempdb";
backupdir_ = "/tempbk";
// set up envs // set up envs
env_ = Env::Default(); db_chroot_env_.reset(NewChrootEnv(Env::Default(), db_chroot));
mock_env_.reset(new MockEnv(env_)); backup_chroot_env_.reset(NewChrootEnv(Env::Default(), backup_chroot));
test_db_env_.reset(new TestEnv(env_)); test_db_env_.reset(new TestEnv(db_chroot_env_.get()));
test_backup_env_.reset(new TestEnv(env_)); test_backup_env_.reset(new TestEnv(backup_chroot_env_.get()));
file_manager_.reset(new FileManager(env_)); file_manager_.reset(new FileManager(backup_chroot_env_.get()));
// set up db options // set up db options
options_.create_if_missing = true; options_.create_if_missing = true;
@ -447,8 +450,7 @@ class BackupableDBTest : public testing::Test {
// Create logger // Create logger
DBOptions logger_options; DBOptions logger_options;
logger_options.env = env_; logger_options.env = db_chroot_env_.get();
logger_options.db_log_dir = backupdir_;
CreateLoggerFromOptions(dbname_, logger_options, &logger_); CreateLoggerFromOptions(dbname_, logger_options, &logger_);
// set up backup db options // set up backup db options
@ -459,7 +461,7 @@ class BackupableDBTest : public testing::Test {
backupable_options_->max_background_operations = 7; backupable_options_->max_background_operations = 7;
// delete old files in db // delete old files in db
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
} }
DB* OpenDB() { DB* OpenDB() {
@ -548,13 +550,13 @@ class BackupableDBTest : public testing::Test {
void DeleteLogFiles() { void DeleteLogFiles() {
std::vector<std::string> delete_logs; std::vector<std::string> delete_logs;
env_->GetChildren(dbname_, &delete_logs); db_chroot_env_->GetChildren(dbname_, &delete_logs);
for (auto f : delete_logs) { for (auto f : delete_logs) {
uint64_t number; uint64_t number;
FileType type; FileType type;
bool ok = ParseFileName(f, &number, &type); bool ok = ParseFileName(f, &number, &type);
if (ok && type == kLogFile) { if (ok && type == kLogFile) {
env_->DeleteFile(dbname_ + "/" + f); db_chroot_env_->DeleteFile(dbname_ + "/" + f);
} }
} }
} }
@ -568,8 +570,8 @@ class BackupableDBTest : public testing::Test {
std::shared_ptr<Logger> logger_; std::shared_ptr<Logger> logger_;
// envs // envs
Env* env_; unique_ptr<Env> db_chroot_env_;
unique_ptr<MockEnv> mock_env_; unique_ptr<Env> backup_chroot_env_;
unique_ptr<TestEnv> test_db_env_; unique_ptr<TestEnv> test_db_env_;
unique_ptr<TestEnv> test_backup_env_; unique_ptr<TestEnv> test_backup_env_;
unique_ptr<FileManager> file_manager_; unique_ptr<FileManager> file_manager_;
@ -642,7 +644,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
// second iter -- don't flush before backup // second iter -- don't flush before backup
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
// delete old data // delete old data
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
bool destroy_data = true; bool destroy_data = true;
// every iteration -- // every iteration --
@ -659,7 +661,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
FillDB(db_.get(), keys_iteration * i, fill_up_to); FillDB(db_.get(), keys_iteration * i, fill_up_to);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0));
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
// ---- make sure it's empty ---- // ---- make sure it's empty ----
DB* db = OpenDB(); DB* db = OpenDB();
@ -687,7 +689,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
const int max_key = keys_iteration * 4 + 10; const int max_key = keys_iteration * 4 + 10;
Random rnd(7); Random rnd(7);
// delete old data // delete old data
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true); OpenDBAndBackupEngine(true);
// write some data, backup, repeat // write some data, backup, repeat
@ -706,7 +708,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
} }
// close and destroy // close and destroy
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
// ---- make sure it's empty ---- // ---- make sure it's empty ----
DB* db = OpenDB(); DB* db = OpenDB();
@ -763,7 +765,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
"/private/1.tmp/CURRENT", "/private/1.tmp/MANIFEST-01", "/private/1.tmp/CURRENT", "/private/1.tmp/MANIFEST-01",
"/private/1.tmp/00011.log", "/meta/1.tmp", "/private/1.tmp/00011.log", "/meta/1.tmp",
"/LATEST_BACKUP.tmp"}; "/LATEST_BACKUP.tmp"};
AppendPath(dbname_ + "_backup", should_have_written); AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written); test_backup_env_->AssertWrittenFiles(should_have_written);
// should write 4 new DB files + LATEST_BACKUP + one meta file // should write 4 new DB files + LATEST_BACKUP + one meta file
@ -784,7 +786,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
"/meta/2.tmp", "/meta/2.tmp",
"/LATEST_BACKUP.tmp" "/LATEST_BACKUP.tmp"
}; };
AppendPath(dbname_ + "_backup", should_have_written); AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written); test_backup_env_->AssertWrittenFiles(should_have_written);
ASSERT_OK(backup_engine_->DeleteBackup(1)); ASSERT_OK(backup_engine_->DeleteBackup(1));
@ -805,41 +807,6 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
} }
// Verify that backup works when the database environment is not the same as
// the backup environment
// TODO(agf): Make all/most tests use different db and backup environments.
// This will probably require more implementation of MockEnv.
// For example, MockEnv::RenameFile() must be able to rename
// directories.
TEST_F(BackupableDBTest, DifferentEnvs) {
test_db_env_.reset(new TestEnv(mock_env_.get()));
options_.env = test_db_env_.get();
OpenDBAndBackupEngine(true, true);
// should write 5 DB files + LATEST_BACKUP + one meta file
test_backup_env_->SetLimitWrittenFiles(7);
test_backup_env_->ClearWrittenFiles();
test_db_env_->SetLimitWrittenFiles(0);
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
"/CURRENT", "/MANIFEST-01" };
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
CloseDBAndBackupEngine();
// try simple backup and verify correctness
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
test_db_env_->SetFilenamesForMockedAttrs({});
AssertBackupConsistency(0, 0, 100, 500);
}
// test various kind of corruptions that may happen: // test various kind of corruptions that may happen:
// 1. Not able to write a file for backup - that backup should fail, // 1. Not able to write a file for backup - that backup should fail,
// everything else should work // everything else should work
@ -893,7 +860,7 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
// assert that we wrote 6 to LATEST_BACKUP // assert that we wrote 6 to LATEST_BACKUP
{ {
std::string latest_backup_contents; std::string latest_backup_contents;
ReadFileToString(env_, backupdir_ + "/LATEST_BACKUP", ReadFileToString(backup_chroot_env_.get(), backupdir_ + "/LATEST_BACKUP",
&latest_backup_contents); &latest_backup_contents);
ASSERT_EQ(std::atol(latest_backup_contents.c_str()), 6); ASSERT_EQ(std::atol(latest_backup_contents.c_str()), 6);
} }
@ -993,7 +960,8 @@ TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {
backupable_options_->destroy_old_data = false; backupable_options_->destroy_old_data = false;
BackupEngineReadOnly* read_only_backup_engine; BackupEngineReadOnly* read_only_backup_engine;
ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_, ASSERT_OK(BackupEngineReadOnly::Open(backup_chroot_env_.get(),
*backupable_options_,
&read_only_backup_engine)); &read_only_backup_engine));
// assert that data from backup 5 is still here (even though LATEST_BACKUP // assert that data from backup 5 is still here (even though LATEST_BACKUP
@ -1160,7 +1128,7 @@ TEST_F(BackupableDBTest, RateLimiting) {
for (const auto& limit : limits) { for (const auto& limit : limits) {
// destroy old data // destroy old data
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
backupable_options_->backup_rate_limit = limit.first; backupable_options_->backup_rate_limit = limit.first;
backupable_options_->restore_rate_limit = limit.second; backupable_options_->restore_rate_limit = limit.second;
@ -1169,9 +1137,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
OpenDBAndBackupEngine(true); OpenDBAndBackupEngine(true);
size_t bytes_written = FillDB(db_.get(), 0, 100000); size_t bytes_written = FillDB(db_.get(), 0, 100000);
auto start_backup = env_->NowMicros(); auto start_backup = db_chroot_env_->NowMicros();
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
auto backup_time = env_->NowMicros() - start_backup; auto backup_time = db_chroot_env_->NowMicros() - start_backup;
auto rate_limited_backup_time = (bytes_written * kMicrosPerSec) / auto rate_limited_backup_time = (bytes_written * kMicrosPerSec) /
backupable_options_->backup_rate_limit; backupable_options_->backup_rate_limit;
ASSERT_GT(backup_time, 0.8 * rate_limited_backup_time); ASSERT_GT(backup_time, 0.8 * rate_limited_backup_time);
@ -1179,9 +1147,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
OpenBackupEngine(); OpenBackupEngine();
auto start_restore = env_->NowMicros(); auto start_restore = db_chroot_env_->NowMicros();
ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_)); ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_));
auto restore_time = env_->NowMicros() - start_restore; auto restore_time = db_chroot_env_->NowMicros() - start_restore;
CloseBackupEngine(); CloseBackupEngine();
auto rate_limited_restore_time = (bytes_written * kMicrosPerSec) / auto rate_limited_restore_time = (bytes_written * kMicrosPerSec) /
backupable_options_->restore_rate_limit; backupable_options_->restore_rate_limit;
@ -1193,21 +1161,21 @@ TEST_F(BackupableDBTest, RateLimiting) {
} }
TEST_F(BackupableDBTest, ReadOnlyBackupEngine) { TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true); OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100); FillDB(db_.get(), 0, 100);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
FillDB(db_.get(), 100, 200); FillDB(db_.get(), 100, 200);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
backupable_options_->destroy_old_data = false; backupable_options_->destroy_old_data = false;
test_backup_env_->ClearWrittenFiles(); test_backup_env_->ClearWrittenFiles();
test_backup_env_->SetLimitDeleteFiles(0); test_backup_env_->SetLimitDeleteFiles(0);
BackupEngineReadOnly* read_only_backup_engine; BackupEngineReadOnly* read_only_backup_engine;
ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_, ASSERT_OK(BackupEngineReadOnly::Open(
&read_only_backup_engine)); db_chroot_env_.get(), *backupable_options_, &read_only_backup_engine));
std::vector<BackupInfo> backup_info; std::vector<BackupInfo> backup_info;
read_only_backup_engine->GetBackupInfo(&backup_info); read_only_backup_engine->GetBackupInfo(&backup_info);
ASSERT_EQ(backup_info.size(), 2U); ASSERT_EQ(backup_info.size(), 2U);
@ -1225,7 +1193,7 @@ TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
} }
TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) { TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true); OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100); FillDB(db_.get(), 0, 100);
bool is_callback_invoked = false; bool is_callback_invoked = false;
@ -1235,14 +1203,14 @@ TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
ASSERT_TRUE(is_callback_invoked); ASSERT_TRUE(is_callback_invoked);
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
} }
TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) { TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true); OpenDBAndBackupEngine(true);
env_->CreateDirIfMissing(backupdir_ + "/shared"); backup_chroot_env_->CreateDirIfMissing(backupdir_ + "/shared");
std::string file_five = backupdir_ + "/shared/000007.sst"; std::string file_five = backupdir_ + "/shared/000007.sst";
std::string file_five_contents = "I'm not really a sst file"; std::string file_five_contents = "I'm not really a sst file";
// this depends on the fact that 00007.sst is the first file created by the DB // this depends on the fact that 00007.sst is the first file created by the DB
@ -1253,7 +1221,8 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok()); ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
std::string new_file_five_contents; std::string new_file_five_contents;
ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents)); ASSERT_OK(ReadFileToString(backup_chroot_env_.get(), file_five,
&new_file_five_contents));
// file 000007.sst was overwritten // file 000007.sst was overwritten
ASSERT_TRUE(new_file_five_contents != file_five_contents); ASSERT_TRUE(new_file_five_contents != file_five_contents);
@ -1301,7 +1270,7 @@ TEST_F(BackupableDBTest, EnvFailures) {
// Verify manifest can roll while a backup is being created with the old // Verify manifest can roll while a backup is being created with the old
// manifest. // manifest.
TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) { TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
options_.max_manifest_file_size = 0; // always rollover manifest for file add options_.max_manifest_file_size = 0; // always rollover manifest for file add
OpenDBAndBackupEngine(true); OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100); FillDB(db_.get(), 0, 100);
@ -1328,12 +1297,12 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
std::string prev_manifest_path = std::string prev_manifest_path =
DescriptorFileName(dbname_, db_impl->TEST_Current_Manifest_FileNo()); DescriptorFileName(dbname_, db_impl->TEST_Current_Manifest_FileNo());
FillDB(db_.get(), 0, 100); FillDB(db_.get(), 0, 100);
ASSERT_OK(env_->FileExists(prev_manifest_path)); ASSERT_OK(db_chroot_env_->FileExists(prev_manifest_path));
ASSERT_OK(db_->Flush(FlushOptions())); ASSERT_OK(db_->Flush(FlushOptions()));
ASSERT_TRUE(env_->FileExists(prev_manifest_path).IsNotFound()); ASSERT_TRUE(db_chroot_env_->FileExists(prev_manifest_path).IsNotFound());
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
AssertBackupConsistency(0, 0, 100); AssertBackupConsistency(0, 0, 100);
} }
@ -1341,9 +1310,10 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
TEST_F(BackupableDBTest, Issue921Test) { TEST_F(BackupableDBTest, Issue921Test) {
BackupEngine* backup_engine; BackupEngine* backup_engine;
backupable_options_->share_table_files = false; backupable_options_->share_table_files = false;
env_->CreateDirIfMissing(backupable_options_->backup_dir); backup_chroot_env_->CreateDirIfMissing(backupable_options_->backup_dir);
backupable_options_->backup_dir += "/new_dir"; backupable_options_->backup_dir += "/new_dir";
ASSERT_OK(BackupEngine::Open(env_, *backupable_options_, &backup_engine)); ASSERT_OK(BackupEngine::Open(backup_chroot_env_.get(), *backupable_options_,
&backup_engine));
delete backup_engine; delete backup_engine;
} }
@ -1368,7 +1338,7 @@ TEST_F(BackupableDBTest, BackupWithMetadata) {
ASSERT_EQ(std::to_string(i), backup_infos[i].app_metadata); ASSERT_EQ(std::to_string(i), backup_infos[i].app_metadata);
} }
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
} }
TEST_F(BackupableDBTest, BinaryMetadata) { TEST_F(BackupableDBTest, BinaryMetadata) {
@ -1386,7 +1356,7 @@ TEST_F(BackupableDBTest, BinaryMetadata) {
ASSERT_EQ(1, backup_infos.size()); ASSERT_EQ(1, backup_infos.size());
ASSERT_EQ(binaryMetadata, backup_infos[0].app_metadata); ASSERT_EQ(binaryMetadata, backup_infos[0].app_metadata);
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
} }
TEST_F(BackupableDBTest, MetadataTooLarge) { TEST_F(BackupableDBTest, MetadataTooLarge) {
@ -1395,7 +1365,7 @@ TEST_F(BackupableDBTest, MetadataTooLarge) {
ASSERT_NOK( ASSERT_NOK(
backup_engine_->CreateNewBackupWithMetadata(db_.get(), largeMetadata)); backup_engine_->CreateNewBackupWithMetadata(db_.get(), largeMetadata));
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
DestroyDB(dbname_, Options()); DestroyDB(dbname_, options_);
} }
} // anon namespace } // anon namespace