diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c7f8168f6..63780bca3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,6 +108,26 @@ jobs: - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain - post-steps + build-linux-mem-env: + machine: + image: ubuntu-1604:202007-01 + resource_class: 2xlarge + steps: + - pre-steps + - install-gflags + - run: MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain + - post-steps + + build-linux-encrypted-env: + machine: + image: ubuntu-1604:202007-01 + resource_class: 2xlarge + steps: + - pre-steps + - install-gflags + - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain + - post-steps + build-linux-shared_lib-alt_namespace-status_checked: machine: image: ubuntu-1604:202007-01 @@ -378,6 +398,12 @@ workflows: build-linux: jobs: - build-linux + build-linux-mem-env: + jobs: + - build-linux-mem-env + build-linux-encrypted-env: + jobs: + - build-linux-encrypted-env build-linux-shared_lib-alt_namespace-status_checked: jobs: - build-linux-shared_lib-alt_namespace-status_checked diff --git a/db/blob/db_blob_basic_test.cc b/db/blob/db_blob_basic_test.cc index d5838af09d..711ef895b9 100644 --- a/db/blob/db_blob_basic_test.cc +++ b/db/blob/db_blob_basic_test.cc @@ -18,7 +18,7 @@ class DBBlobBasicTest : public DBTestBase { }; TEST_F(DBBlobBasicTest, GetBlob) { - Options options; + Options options = GetDefaultOptions(); options.enable_blob_files = true; options.min_blob_size = 0; @@ -45,7 +45,7 @@ TEST_F(DBBlobBasicTest, GetBlob) { } TEST_F(DBBlobBasicTest, GetBlob_CorruptIndex) { - Options options; + Options options = GetDefaultOptions(); options.enable_blob_files = true; options.min_blob_size = 0; @@ -70,7 +70,7 @@ TEST_F(DBBlobBasicTest, GetBlob_CorruptIndex) { TEST_F(DBBlobBasicTest, GetBlob_InlinedTTLIndex) { constexpr uint64_t min_blob_size = 10; - Options options; + Options options = GetDefaultOptions(); options.enable_blob_files = true; options.min_blob_size = min_blob_size; @@ -100,7 +100,7 @@ TEST_F(DBBlobBasicTest, GetBlob_InlinedTTLIndex) { } TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) { - Options options; + Options options = GetDefaultOptions(); options.enable_blob_files = true; options.min_blob_size = 0; @@ -132,11 +132,12 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) { class DBBlobBasicIOErrorTest : public DBBlobBasicTest, public testing::WithParamInterface { protected: - DBBlobBasicIOErrorTest() - : fault_injection_env_(Env::Default()), sync_point_(GetParam()) {} + DBBlobBasicIOErrorTest() : sync_point_(GetParam()) { + fault_injection_env_.reset(new FaultInjectionTestEnv(env_)); + } ~DBBlobBasicIOErrorTest() { Close(); } - FaultInjectionTestEnv fault_injection_env_; + std::unique_ptr fault_injection_env_; std::string sync_point_; }; @@ -147,7 +148,7 @@ INSTANTIATE_TEST_CASE_P(DBBlobBasicTest, DBBlobBasicIOErrorTest, TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) { Options options; - options.env = &fault_injection_env_; + options.env = fault_injection_env_.get(); options.enable_blob_files = true; options.min_blob_size = 0; @@ -161,8 +162,8 @@ TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) { ASSERT_OK(Flush()); SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { - fault_injection_env_.SetFilesystemActive(false, - Status::IOError(sync_point_)); + fault_injection_env_->SetFilesystemActive(false, + Status::IOError(sync_point_)); }); SyncPoint::GetInstance()->EnableProcessing(); diff --git a/db/blob/db_blob_index_test.cc b/db/blob/db_blob_index_test.cc index f7525c1d8e..6a55b9769c 100644 --- a/db/blob/db_blob_index_test.cc +++ b/db/blob/db_blob_index_test.cc @@ -103,6 +103,7 @@ class DBBlobIndexTest : public DBTestBase { Options GetTestOptions() { Options options; + options.env = CurrentOptions().env; options.create_if_missing = true; options.num_levels = 2; options.disable_auto_compactions = true; diff --git a/db/corruption_test.cc b/db/corruption_test.cc index 97acc3ab51..ef5dbba48d 100644 --- a/db/corruption_test.cc +++ b/db/corruption_test.cc @@ -184,7 +184,7 @@ class CorruptionTest : public testing::Test { } ASSERT_TRUE(!fname.empty()) << filetype; - test::CorruptFile(fname, offset, bytes_to_corrupt); + ASSERT_OK(test::CorruptFile(&env_, fname, offset, bytes_to_corrupt)); } // corrupts exactly one file at level `level`. if no file found at level, @@ -194,7 +194,8 @@ class CorruptionTest : public testing::Test { db_->GetLiveFilesMetaData(&metadata); for (const auto& m : metadata) { if (m.level == level) { - test::CorruptFile(dbname_ + "/" + m.name, offset, bytes_to_corrupt); + ASSERT_OK(test::CorruptFile(&env_, dbname_ + "/" + m.name, offset, + bytes_to_corrupt)); return; } } @@ -529,7 +530,8 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) { ImmutableCFOptions(options_), kRangeDelBlock, &range_del_handle)); ASSERT_OK(TryReopen()); - test::CorruptFile(filename, static_cast(range_del_handle.offset()), 1); + ASSERT_OK(test::CorruptFile(&env_, filename, + static_cast(range_del_handle.offset()), 1)); ASSERT_TRUE(TryReopen().IsCorruption()); } diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index f60144d329..dda217db60 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -12,6 +12,7 @@ #include "db/db_test_util.h" #include "port/stack_trace.h" +#include "rocksdb/flush_block_policy.h" #include "rocksdb/merge_operator.h" #include "rocksdb/perf_context.h" #include "rocksdb/utilities/debug.h" @@ -409,8 +410,7 @@ TEST_F(DBBasicTest, CheckLock) { Status s = DB::Open(options, dbname_, &localdb); ASSERT_NOK(s); #ifdef OS_LINUX - ASSERT_TRUE(s.ToString().find("lock hold by current process") != - std::string::npos); + ASSERT_TRUE(s.ToString().find("lock ") != std::string::npos); #endif // OS_LINUX } while (ChangeCompactOptions()); } @@ -1875,6 +1875,7 @@ TEST_F(DBBasicTest, MultiGetStats) { Options options; options.create_if_missing = true; options.disable_auto_compactions = true; + options.env = env_; options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); BlockBasedTableOptions table_options; table_options.block_size = 1; @@ -1884,7 +1885,7 @@ TEST_F(DBBasicTest, MultiGetStats) { table_options.no_block_cache = true; table_options.cache_index_and_filter_blocks = false; table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, options); int total_keys = 2000; @@ -2168,7 +2169,7 @@ TEST_F(DBBasicTest, MultiGetIOBufferOverrun) { table_options.block_size = 16 * 1024; ASSERT_TRUE(table_options.block_size > BlockBasedTable::kMultiGetReadStackBufSize); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); std::string zero_str(128, '\0'); @@ -2549,7 +2550,7 @@ class DBBasicTestMultiGet : public DBTestBase { table_options.block_cache_compressed = compressed_cache_; table_options.flush_block_policy_factory.reset( new MyFlushBlockPolicyFactory()); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); if (!compression_enabled_) { options.compression = kNoCompression; } else { @@ -2976,7 +2977,7 @@ class DeadlineFS : public FileSystemWrapper { // or to simply delay but return success anyway. The latter mimics the // behavior of PosixFileSystem, which does not enforce any timeout explicit DeadlineFS(SpecialEnv* env, bool error_on_delay) - : FileSystemWrapper(FileSystem::Default()), + : FileSystemWrapper(env->GetFileSystem()), deadline_(std::chrono::microseconds::zero()), io_timeout_(std::chrono::microseconds::zero()), env_(env), @@ -3151,7 +3152,7 @@ TEST_F(DBBasicTestMultiGetDeadline, MultiGetDeadlineExceeded) { std::shared_ptr cache = NewLRUCache(1048576); BlockBasedTableOptions table_options; table_options.block_cache = cache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.env = env.get(); SetTimeElapseOnlySleepOnReopen(&options); ReopenWithColumnFamilies(GetCFNames(), options); diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index d461024618..3d4bd3a9d3 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -1730,6 +1730,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterUpperBound) { int using_full_builder = bfp_impl != BFP::kDeprecatedBlock; Options options; options.create_if_missing = true; + options.env = CurrentOptions().env; options.prefix_extractor.reset(NewCappedPrefixTransform(4)); options.disable_auto_compactions = true; options.statistics = CreateDBStatistics(); @@ -1860,6 +1861,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) { for (auto bfp_impl : BFP::kAllFixedImpls) { int using_full_builder = bfp_impl != BFP::kDeprecatedBlock; Options options; + options.env = CurrentOptions().env; options.create_if_missing = true; options.prefix_extractor.reset(NewFixedPrefixTransform(1)); options.disable_auto_compactions = true; @@ -2052,6 +2054,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) { TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) { for (auto bfp_impl : BFP::kAllFixedImpls) { Options options; + options.env = CurrentOptions().env; options.create_if_missing = true; options.prefix_extractor.reset(NewFixedPrefixTransform(1)); options.disable_auto_compactions = true; diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index d90c31bda1..3d24945eb3 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -3270,7 +3270,7 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) { table_options.block_cache = NewLRUCache(64 << 20); // 64MB table_options.cache_index_and_filter_blocks = true; table_options.pin_l0_filter_and_index_blocks_in_cache = true; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); DestroyAndReopen(options); @@ -5023,7 +5023,7 @@ TEST_F(DBCompactionTest, ManualCompactionFailsInReadOnlyMode) { // is in read-only mode. Verify it now at least returns, despite failing. const int kNumL0Files = 4; std::unique_ptr mock_env( - new FaultInjectionTestEnv(Env::Default())); + new FaultInjectionTestEnv(env_)); Options opts = CurrentOptions(); opts.disable_auto_compactions = true; opts.env = mock_env.get(); @@ -5250,7 +5250,7 @@ TEST_F(DBCompactionTest, ConsistencyFailTest2) { options.level0_file_num_compaction_trigger = 2; BlockBasedTableOptions bbto; bbto.block_size = 400; // small block size - options.table_factory.reset(new BlockBasedTableFactory(bbto)); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); DestroyAndReopen(options); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( diff --git a/db/db_encryption_test.cc b/db/db_encryption_test.cc index dddc833071..2322d08671 100644 --- a/db/db_encryption_test.cc +++ b/db/db_encryption_test.cc @@ -18,6 +18,13 @@ class DBEncryptionTest : public DBTestBase { public: DBEncryptionTest() : DBTestBase("/db_encryption_test", /*env_do_fsync=*/true) {} + Env* GetTargetEnv() { + if (encrypted_env_ != nullptr) { + return (static_cast(encrypted_env_))->target(); + } else { + return env_; + } + } }; #ifndef ROCKSDB_LITE @@ -34,20 +41,20 @@ TEST_F(DBEncryptionTest, CheckEncrypted) { auto status = env_->GetChildren(dbname_, &fileNames); ASSERT_OK(status); - auto defaultEnv = Env::Default(); + Env* target = GetTargetEnv(); int hits = 0; for (auto it = fileNames.begin() ; it != fileNames.end(); ++it) { - if ((*it == "..") || (*it == ".")) { + if ((*it == "..") || (*it == ".") || (*it == "LOCK")) { continue; } auto filePath = dbname_ + "/" + *it; std::unique_ptr seqFile; auto envOptions = EnvOptions(CurrentOptions()); - status = defaultEnv->NewSequentialFile(filePath, &seqFile, envOptions); + status = target->NewSequentialFile(filePath, &seqFile, envOptions); ASSERT_OK(status); uint64_t fileSize; - status = defaultEnv->GetFileSize(filePath, &fileSize); + status = target->GetFileSize(filePath, &fileSize); ASSERT_OK(status); std::string scratch; @@ -85,7 +92,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) { } TEST_F(DBEncryptionTest, ReadEmptyFile) { - auto defaultEnv = Env::Default(); + auto defaultEnv = GetTargetEnv(); // create empty file for reading it back in later auto envOptions = EnvOptions(CurrentOptions()); diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index 1789ce44e9..eeaa691237 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -450,6 +450,7 @@ TEST_F(DBFlushTest, FlushWithBlob) { constexpr uint64_t min_blob_size = 10; Options options; + options.env = CurrentOptions().env; options.enable_blob_files = true; options.min_blob_size = min_blob_size; options.disable_auto_compactions = true; diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index ace3e35428..f7a074d680 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -637,7 +637,7 @@ Status DBImpl::CloseHelper() { if (immutable_db_options_.info_log && own_info_log_) { Status s = immutable_db_options_.info_log->Close(); - if (!s.ok() && ret.ok()) { + if (!s.ok() && !s.IsNotSupported() && ret.ok()) { ret = s; } } diff --git a/db/db_impl/db_secondary_test.cc b/db/db_impl/db_secondary_test.cc index 5f7693366e..bad0153a66 100644 --- a/db/db_impl/db_secondary_test.cc +++ b/db/db_impl/db_secondary_test.cc @@ -883,6 +883,7 @@ TEST_F(DBSecondaryTest, StartFromInconsistent) { }); SyncPoint::GetInstance()->EnableProcessing(); Options options1; + options1.env = env_; Status s = TryOpenSecondary(options1); ASSERT_TRUE(s.IsCorruption()); } @@ -894,6 +895,7 @@ TEST_F(DBSecondaryTest, InconsistencyDuringCatchUp) { ASSERT_OK(Flush()); Options options1; + options1.env = env_; OpenSecondary(options1); { diff --git a/db/db_memtable_test.cc b/db/db_memtable_test.cc index 99763e3516..8706b690bf 100644 --- a/db/db_memtable_test.cc +++ b/db/db_memtable_test.cc @@ -316,6 +316,7 @@ TEST_F(DBMemTableTest, InsertWithHint) { TEST_F(DBMemTableTest, ColumnFamilyId) { // Verifies MemTableRepFactory is told the right column family id. Options options; + options.env = CurrentOptions().env; options.allow_concurrent_memtable_write = false; options.create_if_missing = true; options.memtable_factory.reset(new MockMemTableRepFactory()); diff --git a/db/db_options_test.cc b/db/db_options_test.cc index a49b769397..a950cb56e2 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -81,6 +81,7 @@ class DBOptionsTest : public DBTestBase { TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) { Options options; + options.env = env_; options.track_and_verify_wals_in_manifest = true; ImmutableDBOptions db_options(options); @@ -621,6 +622,7 @@ TEST_F(DBOptionsTest, MaxOpenFilesChange) { TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) { Options options; + options.env = CurrentOptions().env; options.delayed_write_rate = 0; Reopen(options); ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); @@ -632,6 +634,7 @@ TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) { TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) { Options options; + options.env = CurrentOptions().env; options.compaction_style = kCompactionStyleUniversal; options.ttl = 0; @@ -661,6 +664,7 @@ TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) { TEST_F(DBOptionsTest, SanitizeTtlDefault) { Options options; + options.env = CurrentOptions().env; Reopen(options); ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); @@ -677,6 +681,7 @@ TEST_F(DBOptionsTest, SanitizeTtlDefault) { TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) { Options options; options.compaction_style = kCompactionStyleFIFO; + options.env = CurrentOptions().env; options.ttl = 0; Reopen(options); ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); @@ -702,6 +707,7 @@ TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) { TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { Options options; + options.env = CurrentOptions().env; options.compaction_style = kCompactionStyleFIFO; options.write_buffer_size = 10 << 10; // 10KB options.arena_block_size = 4096; @@ -841,6 +847,7 @@ TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) { options.compaction_style = kCompactionStyleFIFO; options.write_buffer_size = 10 << 10; // 10KB options.create_if_missing = true; + options.env = CurrentOptions().env; ASSERT_OK(TryReopen(options)); @@ -894,6 +901,7 @@ TEST_F(DBOptionsTest, ChangeCompression) { options.bottommost_compression = CompressionType::kNoCompression; options.bottommost_compression_opts.level = 2; options.bottommost_compression_opts.parallel_threads = 1; + options.env = CurrentOptions().env; ASSERT_OK(TryReopen(options)); diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index ff6ded37fa..d3333fa933 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -336,7 +336,7 @@ TEST_F(DBPropertiesTest, AggregatedTableProperties) { table_options.filter_policy.reset( NewBloomFilterPolicy(kBloomBitsPerKey, false)); table_options.block_size = 1024; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); DestroyAndReopen(options); @@ -536,7 +536,7 @@ TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) { table_options.filter_policy.reset( NewBloomFilterPolicy(kBloomBitsPerKey, false)); table_options.block_size = 1024; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); DestroyAndReopen(options); @@ -1414,7 +1414,7 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) { } TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) { - Options options; + Options options = CurrentOptions(); Reopen(options); ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Delete("foo")); @@ -1524,6 +1524,7 @@ TEST_F(DBPropertiesTest, SstFilesSize) { std::shared_ptr listener = std::make_shared(); Options options; + options.env = CurrentOptions().env; options.disable_auto_compactions = true; options.listeners.push_back(listener); Reopen(options); @@ -1608,6 +1609,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) { Options options; uint64_t value; + options.env = CurrentOptions().env; + // Block cache properties are not available for tables other than // block-based table. options.table_factory.reset(NewPlainTableFactory()); diff --git a/db/db_sst_test.cc b/db/db_sst_test.cc index 2287c2425a..3ccb57baa5 100644 --- a/db/db_sst_test.cc +++ b/db/db_sst_test.cc @@ -141,6 +141,7 @@ TEST_F(DBSSTTest, SkipCheckingSSTFileSizesOnDBOpen) { // Just open the DB with the option set to true and check that we don't crash. Options options; + options.env = env_; options.skip_checking_sst_file_sizes_on_db_open = true; Reopen(options); @@ -519,7 +520,7 @@ TEST_P(DBWALTestWithParam, WALTrashCleanupOnOpen) { bool fake_log_delete; }; - std::unique_ptr env(new MyEnv(Env::Default())); + std::unique_ptr env(new MyEnv(env_)); Destroy(last_options_); env->set_fake_log_delete(true); diff --git a/db/db_test.cc b/db/db_test.cc index 0ea4c94840..d733745110 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2391,6 +2391,7 @@ TEST_F(DBTest, PurgeInfoLogs) { Options options = CurrentOptions(); options.keep_log_file_num = 5; options.create_if_missing = true; + options.env = env_; for (int mode = 0; mode <= 1; mode++) { if (mode == 1) { options.db_log_dir = dbname_ + "_logs"; @@ -3947,6 +3948,7 @@ TEST_F(DBTest, WriteSingleThreadEntry) { TEST_F(DBTest, ConcurrentFlushWAL) { const size_t cnt = 100; Options options; + options.env = env_; WriteOptions wopt; ReadOptions ropt; for (bool two_write_queues : {false, true}) { @@ -4615,6 +4617,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) { Random rnd(301); Options options; + options.env = env_; options.create_if_missing = true; options.db_write_buffer_size = 20480; options.write_buffer_size = 20480; @@ -5017,6 +5020,7 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) { Options options; options.ttl = 0; options.create_if_missing = true; + options.env = env_; DestroyAndReopen(options); // Initial defaults @@ -5078,6 +5082,7 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) { TEST_F(DBTest, DynamicUniversalCompactionOptions) { Options options; options.create_if_missing = true; + options.env = env_; DestroyAndReopen(options); // Initial defaults diff --git a/db/db_test2.cc b/db/db_test2.cc index 4c2cce1550..b3875f8456 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -104,6 +104,7 @@ class TestReadOnlyWithCompressedCache TEST_P(TestReadOnlyWithCompressedCache, ReadOnlyWithCompressedCache) { if (use_mmap_ && !IsMemoryMappedAccessSupported()) { + ROCKSDB_GTEST_SKIP("Test requires MMAP support"); return; } ASSERT_OK(Put("foo", "bar")); @@ -291,7 +292,7 @@ TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) { BlockBasedTableOptions table_options; table_options.cache_index_and_filter_blocks = true; table_options.filter_policy.reset(NewBloomFilterPolicy(20)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, options); Put(1, "a", "begin"); @@ -1344,7 +1345,7 @@ TEST_F(DBTest2, PresetCompressionDictLocality) { options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry; BlockBasedTableOptions table_options; table_options.cache_index_and_filter_blocks = true; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); Random rnd(301); @@ -1470,7 +1471,7 @@ TEST_P(CompressionFailuresTest, CompressionFailures) { BlockBasedTableOptions table_options; table_options.block_size = 512; table_options.verify_compression = true; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.compression = compression_type_; options.compression_opts.parallel_threads = compression_parallel_threads_; @@ -1808,7 +1809,7 @@ class PinL0IndexAndFilterBlocksTest table_options.cache_index_and_filter_blocks = true; table_options.pin_l0_filter_and_index_blocks_in_cache = true; table_options.filter_policy.reset(NewBloomFilterPolicy(20)); - options->table_factory.reset(new BlockBasedTableFactory(table_options)); + options->table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, *options); Put(1, "a", "begin"); @@ -1848,7 +1849,7 @@ TEST_P(PinL0IndexAndFilterBlocksTest, table_options.cache_index_and_filter_blocks = true; table_options.pin_l0_filter_and_index_blocks_in_cache = true; table_options.filter_policy.reset(NewBloomFilterPolicy(20)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, options); ASSERT_OK(Put(1, "key", "val")); @@ -2485,26 +2486,30 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) { { const int kIdBufLen = 100; char id_buf[kIdBufLen]; + Status s = Status::NotSupported(); #ifndef OS_WIN // You can't open a directory on windows using random access file std::unique_ptr file; - ASSERT_OK(env_->NewRandomAccessFile(dbname_, &file, EnvOptions())); - if (file->GetUniqueId(id_buf, kIdBufLen) == 0) { - // fs holding db directory doesn't support getting a unique file id, - // this means that running this test will fail because lru_cache will load - // the blocks again regardless of them being already in the cache - return; - } -#else - std::unique_ptr dir; - ASSERT_OK(env_->NewDirectory(dbname_, &dir)); - if (dir->GetUniqueId(id_buf, kIdBufLen) == 0) { - // fs holding db directory doesn't support getting a unique file id, - // this means that running this test will fail because lru_cache will load - // the blocks again regardless of them being already in the cache - return; + s = env_->NewRandomAccessFile(dbname_, &file, EnvOptions()); + if (s.ok()) { + if (file->GetUniqueId(id_buf, kIdBufLen) == 0) { + // fs holding db directory doesn't support getting a unique file id, + // this means that running this test will fail because lru_cache will + // load the blocks again regardless of them being already in the cache + return; + } } #endif + if (!s.ok()) { + std::unique_ptr dir; + ASSERT_OK(env_->NewDirectory(dbname_, &dir)); + if (dir->GetUniqueId(id_buf, kIdBufLen) == 0) { + // fs holding db directory doesn't support getting a unique file id, + // this means that running this test will fail because lru_cache will + // load the blocks again regardless of them being already in the cache + return; + } + } } uint32_t bytes_per_bit[2] = {1, 16}; for (size_t k = 0; k < 2; k++) { @@ -3297,7 +3302,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) { BlockBasedTableOptions bbto; bbto.block_size = 16384; bbto.no_block_cache = true; - options.table_factory.reset(new BlockBasedTableFactory(bbto)); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); DestroyAndReopen(options); for (int i = 0; i < kNumL0Files; ++i) { @@ -3342,6 +3347,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) { // is on levels higher than the new num_levels. TEST_F(DBTest2, ReduceLevel) { Options options; + options.env = env_; options.disable_auto_compactions = true; options.num_levels = 7; Reopen(options); @@ -3370,6 +3376,7 @@ TEST_F(DBTest2, ReadCallbackTest) { Options options; options.disable_auto_compactions = true; options.num_levels = 7; + options.env = env_; Reopen(options); std::vector snapshots; // Try to create a db with multiple layers and a memtable @@ -3629,7 +3636,9 @@ TEST_F(DBTest2, TraceAndReplay) { column_families.push_back( ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); std::vector handles; - ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2)); + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); env_->SleepForMicroseconds(100); // Verify that the keys don't already exist @@ -3704,7 +3713,9 @@ TEST_F(DBTest2, TraceWithLimit) { column_families.push_back( ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); std::vector handles; - ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2)); + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); env_->SleepForMicroseconds(100); // Verify that the keys don't already exist @@ -3772,7 +3783,9 @@ TEST_F(DBTest2, TraceWithSampling) { column_families.push_back( ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); std::vector handles; - ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2)); + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); env_->SleepForMicroseconds(100); ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); @@ -3872,7 +3885,9 @@ TEST_F(DBTest2, TraceWithFilter) { column_families.push_back( ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); std::vector handles; - ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2)); + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); env_->SleepForMicroseconds(100); // Verify that the keys don't already exist @@ -3918,7 +3933,7 @@ TEST_F(DBTest2, TraceWithFilter) { handles.clear(); DB* db3 = nullptr; - ASSERT_OK(DB::Open(DBOptions(), dbname3, column_families, &handles, &db3)); + ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3)); env_->SleepForMicroseconds(100); // Verify that the keys don't already exist @@ -3974,6 +3989,11 @@ TEST_F(DBTest2, TraceWithFilter) { TEST_F(DBTest2, PinnableSliceAndMmapReads) { Options options = CurrentOptions(); + options.env = env_; + if (options.env != Env::Default()) { + ROCKSDB_GTEST_SKIP("Test requires default environment"); + return; + } options.allow_mmap_reads = true; options.max_open_files = 100; options.compression = kNoCompression; @@ -4026,7 +4046,7 @@ TEST_F(DBTest2, DISABLED_IteratorPinnedMemory) { bbto.cache_index_and_filter_blocks = false; bbto.block_cache = NewLRUCache(100000); bbto.block_size = 400; // small block size - options.table_factory.reset(new BlockBasedTableFactory(bbto)); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); Reopen(options); Random rnd(301); @@ -4252,6 +4272,7 @@ TEST_F(DBTest2, TestCompactFiles) { SyncPoint::GetInstance()->EnableProcessing(); Options options; + options.env = env_; options.num_levels = 2; options.disable_auto_compactions = true; Reopen(options); @@ -4807,6 +4828,7 @@ TEST_F(DBTest2, BlockBasedTablePrefixIndexSeekForPrev) { TEST_F(DBTest2, PartitionedIndexPrefetchFailure) { Options options = last_options_; + options.env = env_; options.max_open_files = 20; BlockBasedTableOptions bbto; bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; diff --git a/db/db_test_util.cc b/db/db_test_util.cc index ccfd1c8f96..ecd0d7c58a 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -331,7 +331,7 @@ Options DBTestBase::CurrentOptions( return GetOptions(option_config_, default_options, options_override); } -Options DBTestBase::GetDefaultOptions() { +Options DBTestBase::GetDefaultOptions() const { Options options; options.write_buffer_size = 4090 * 4096; options.target_file_size_base = 2 * 1024 * 1024; @@ -339,6 +339,7 @@ Options DBTestBase::GetDefaultOptions() { options.max_open_files = 5000; options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; options.compaction_pri = CompactionPri::kByCompensatedSize; + options.env = env_; return options; } @@ -367,28 +368,28 @@ Options DBTestBase::GetOptions( options.unordered_write = false; break; case kPlainTableFirstBytePrefix: - options.table_factory.reset(new PlainTableFactory()); + options.table_factory.reset(NewPlainTableFactory()); options.prefix_extractor.reset(NewFixedPrefixTransform(1)); options.allow_mmap_reads = can_allow_mmap; options.max_sequential_skip_in_iterations = 999999; set_block_based_table_factory = false; break; case kPlainTableCappedPrefix: - options.table_factory.reset(new PlainTableFactory()); + options.table_factory.reset(NewPlainTableFactory()); options.prefix_extractor.reset(NewCappedPrefixTransform(8)); options.allow_mmap_reads = can_allow_mmap; options.max_sequential_skip_in_iterations = 999999; set_block_based_table_factory = false; break; case kPlainTableCappedPrefixNonMmap: - options.table_factory.reset(new PlainTableFactory()); + options.table_factory.reset(NewPlainTableFactory()); options.prefix_extractor.reset(NewCappedPrefixTransform(8)); options.allow_mmap_reads = false; options.max_sequential_skip_in_iterations = 999999; set_block_based_table_factory = false; break; case kPlainTableAllBytesPrefix: - options.table_factory.reset(new PlainTableFactory()); + options.table_factory.reset(NewPlainTableFactory()); options.prefix_extractor.reset(NewNoopTransform()); options.allow_mmap_reads = can_allow_mmap; options.max_sequential_skip_in_iterations = 999999; @@ -704,9 +705,9 @@ Status DBTestBase::TryReopen(const Options& options) { // Note: operator= is an unsafe approach here since it destructs // std::shared_ptr in the same order of their creation, in contrast to // destructors which destructs them in the opposite order of creation. One - // particular problme is that the cache destructor might invoke callback + // particular problem is that the cache destructor might invoke callback // functions that use Option members such as statistics. To work around this - // problem, we manually call destructor of table_facotry which eventually + // problem, we manually call destructor of table_factory which eventually // clears the block cache. last_options_ = options; MaybeInstallTimeElapseOnlySleep(options); diff --git a/db/db_test_util.h b/db/db_test_util.h index 7a2fbfc70c..2a511ae489 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -949,10 +949,13 @@ class DBTestBase : public testing::Test { const anon::OptionsOverride& options_override = anon::OptionsOverride()) const; - static Options GetDefaultOptions(); + Options GetDefaultOptions() const; - Options GetOptions(int option_config, - const Options& default_options = GetDefaultOptions(), + Options GetOptions(int option_config) const { + return GetOptions(option_config, GetDefaultOptions()); + } + + Options GetOptions(int option_config, const Options& default_options, const anon::OptionsOverride& options_override = anon::OptionsOverride()) const; diff --git a/db/db_universal_compaction_test.cc b/db/db_universal_compaction_test.cc index 119fc66c50..d7f93dc349 100644 --- a/db/db_universal_compaction_test.cc +++ b/db/db_universal_compaction_test.cc @@ -2129,7 +2129,7 @@ TEST_F(DBTestUniversalCompaction2, IngestBehind) { TEST_F(DBTestUniversalCompaction2, PeriodicCompactionDefault) { Options options; options.compaction_style = kCompactionStyleUniversal; - + options.env = env_; KeepFilterFactory* filter = new KeepFilterFactory(true); options.compaction_filter_factory.reset(filter); Reopen(options); diff --git a/db/db_wal_test.cc b/db/db_wal_test.cc index 56d9b35ae9..d0f3787b8c 100644 --- a/db/db_wal_test.cc +++ b/db/db_wal_test.cc @@ -1181,32 +1181,13 @@ class RecoveryTestHelper { test->Close(); #endif if (trunc) { - ASSERT_EQ(0, truncate(fname.c_str(), static_cast(size * off))); + ASSERT_OK( + test::TruncateFile(env, fname, static_cast(size * off))); } else { - InduceCorruption(fname, static_cast(size * off + 8), - static_cast(size * len)); + ASSERT_OK(test::CorruptFile(env, fname, static_cast(size * off + 8), + static_cast(size * len), false)); } } - - // Overwrite data with 'a' from offset for length len - static void InduceCorruption(const std::string& filename, size_t offset, - size_t len) { - ASSERT_GT(len, 0U); - - int fd = open(filename.c_str(), O_RDWR); - - // On windows long is 32-bit - ASSERT_LE(offset, std::numeric_limits::max()); - - ASSERT_GT(fd, 0); - ASSERT_EQ(offset, lseek(fd, static_cast(offset), SEEK_SET)); - - void* buf = alloca(len); - memset(buf, 'b', len); - ASSERT_EQ(len, write(fd, buf, static_cast(len))); - - close(fd); - } }; class DBWALTestWithParams @@ -1328,8 +1309,7 @@ TEST_F(DBWALTest, kPointInTimeRecoveryCFConsistency) { ASSERT_OK(Put(1, "key3", "val3")); // Corrupt WAL at location of key3 - RecoveryTestHelper::InduceCorruption( - fname, static_cast(offset_to_corrupt), static_cast(4)); + test::CorruptFile(env, fname, static_cast(offset_to_corrupt), 4, false); ASSERT_OK(Put(2, "key4", "val4")); ASSERT_OK(Put(1, "key5", "val5")); Flush(2); @@ -1717,6 +1697,10 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithoutFlush) { constexpr size_t kKB = 1024; Options options = CurrentOptions(); options.avoid_flush_during_recovery = true; + if (options.env != Env::Default()) { + ROCKSDB_GTEST_SKIP("Test requires default environment"); + return; + } // Test fallocate support of running file system. // Skip this test if fallocate is not supported. std::string fname_test_fallocate = dbname_ + "/preallocate_testfile"; diff --git a/db/db_write_test.cc b/db/db_write_test.cc index b238960506..945b08eda8 100644 --- a/db/db_write_test.cc +++ b/db/db_write_test.cc @@ -260,7 +260,7 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) { constexpr int kNumThreads = 5; std::unique_ptr mock_env( - new FaultInjectionTestEnv(Env::Default())); + new FaultInjectionTestEnv(env_)); Options options = GetOptions(); options.env = mock_env.get(); Reopen(options); @@ -329,7 +329,7 @@ TEST_P(DBWriteTest, ManualWalFlushInEffect) { TEST_P(DBWriteTest, IOErrorOnWALWriteTriggersReadOnlyMode) { std::unique_ptr mock_env( - new FaultInjectionTestEnv(Env::Default())); + new FaultInjectionTestEnv(env_)); Options options = GetOptions(); options.env = mock_env.get(); Reopen(options); @@ -361,7 +361,7 @@ TEST_P(DBWriteTest, IOErrorOnWALWriteTriggersReadOnlyMode) { TEST_P(DBWriteTest, IOErrorOnSwitchMemtable) { Random rnd(301); std::unique_ptr mock_env( - new FaultInjectionTestEnv(Env::Default())); + new FaultInjectionTestEnv(env_)); Options options = GetOptions(); options.env = mock_env.get(); options.writable_file_max_buffer_size = 4 * 1024 * 1024; diff --git a/db/error_handler_fs_test.cc b/db/error_handler_fs_test.cc index 895c878ab6..34c2f4cbb2 100644 --- a/db/error_handler_fs_test.cc +++ b/db/error_handler_fs_test.cc @@ -25,7 +25,10 @@ namespace ROCKSDB_NAMESPACE { class DBErrorHandlingFSTest : public DBTestBase { public: DBErrorHandlingFSTest() - : DBTestBase("/db_error_handling_fs_test", /*env_do_fsync=*/true) {} + : DBTestBase("/db_error_handling_fs_test", /*env_do_fsync=*/true) { + fault_fs_.reset(new FaultInjectionTestFS(env_->GetFileSystem())); + fault_env_.reset(new CompositeEnvWrapper(env_, fault_fs_)); + } std::string GetManifestNameFromLiveFiles() { std::vector live_files; @@ -44,21 +47,9 @@ class DBErrorHandlingFSTest : public DBTestBase { } return ""; } -}; -class DBErrorHandlingFS : public FileSystemWrapper { - public: - DBErrorHandlingFS() - : FileSystemWrapper(FileSystem::Default()), - trig_no_space(false), - trig_io_error(false) {} - - void SetTrigNoSpace() { trig_no_space = true; } - void SetTrigIoError() { trig_io_error = true; } - - private: - bool trig_no_space; - bool trig_io_error; + std::shared_ptr fault_fs_; + std::unique_ptr fault_env_; }; class ErrorHandlerFSListener : public EventListener { @@ -161,13 +152,10 @@ class ErrorHandlerFSListener : public EventListener { }; TEST_F(DBErrorHandlingFSTest, FLushWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); Status s; @@ -177,13 +165,13 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteError) { ASSERT_OK(Put(Key(0), "val")); SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); @@ -193,13 +181,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteError) { } TEST_F(DBErrorHandlingFSTest, FLushWritRetryableError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 0; @@ -214,42 +199,42 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableError) { ASSERT_OK(Put(Key(1), "val1")); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); - ASSERT_EQ(s, Status::OK()); + ASSERT_OK(s); Reopen(options); ASSERT_EQ("val1", Get(Key(1))); ASSERT_OK(Put(Key(2), "val2")); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeSyncTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); - ASSERT_EQ(s, Status::OK()); + ASSERT_OK(s); Reopen(options); ASSERT_EQ("val2", Get(Key(2))); ASSERT_OK(Put(Key(3), "val3")); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeCloseTableFile", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); - ASSERT_EQ(s, Status::OK()); + ASSERT_OK(s); Reopen(options); ASSERT_EQ("val3", Get(Key(3))); @@ -257,13 +242,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableError) { } TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 0; @@ -280,14 +262,14 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { ASSERT_OK(Put(Key(1), "val1", wo)); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_OK(Put(Key(2), "val2", wo)); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); ASSERT_EQ("val2", Get(Key(2))); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); ASSERT_EQ("val1", Get(Key(1))); @@ -302,13 +284,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { } TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError2) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 0; @@ -326,14 +305,14 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError2) { ASSERT_OK(Put(Key(1), "val1", wo)); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeSyncTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_OK(Put(Key(2), "val2", wo)); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); ASSERT_EQ("val2", Get(Key(2))); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); ASSERT_EQ("val1", Get(Key(1))); @@ -348,13 +327,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError2) { } TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError3) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 0; @@ -372,14 +348,14 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError3) { ASSERT_OK(Put(Key(1), "val1", wo)); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeCloseTableFile", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_OK(Put(Key(2), "val2", wo)); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); ASSERT_EQ("val2", Get(Key(2))); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); ASSERT_EQ("val1", Get(Key(1))); @@ -394,13 +370,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError3) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); Status s; @@ -416,14 +389,15 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { ASSERT_OK(Put(Key(1), "val")); SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); @@ -437,13 +411,10 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 0; @@ -463,13 +434,13 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { ASSERT_OK(Put(Key(1), "val")); SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); @@ -483,13 +454,10 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); Status s; @@ -505,17 +473,18 @@ TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { ASSERT_OK(Put(Key(1), "val")); SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); // This Resume() will attempt to create a new manifest file and fail again s = dbfull()->Resume(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -533,13 +502,14 @@ TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -573,8 +543,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", [&](void*) { if (fail_manifest.load()) { - fault_fs->SetFilesystemActive(false, - IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); } }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -587,7 +557,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { TEST_SYNC_POINT("CompactionManifestWriteError:0"); // Clear all errors so when the compaction is retried, it will succeed - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("CompactionManifestWriteError:1"); TEST_SYNC_POINT("CompactionManifestWriteError:2"); @@ -606,13 +576,10 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -649,7 +616,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", [&](void*) { if (fail_manifest.load()) { - fault_fs->SetFilesystemActive(false, error_msg); + fault_fs_->SetFilesystemActive(false, error_msg); } }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -664,7 +631,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { s = dbfull()->TEST_WaitForCompact(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); s = dbfull()->Resume(); @@ -681,13 +648,10 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -707,7 +671,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { "BackgroundCallCompaction:0"}}); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "BackgroundCallCompaction:0", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -718,20 +683,17 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { s = dbfull()->TEST_WaitForCompact(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); Destroy(options); } TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -754,7 +716,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableError) { "BackgroundCallCompaction:0"}}); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "CompactionJob::OpenCompactionOutputFile", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); ASSERT_OK(Put(Key(1), "val")); @@ -764,7 +726,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableError) { s = dbfull()->TEST_WaitForCompact(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); s = dbfull()->Resume(); @@ -773,11 +735,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, CorruptionError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; Status s; @@ -793,8 +752,8 @@ TEST_F(DBErrorHandlingFSTest, CorruptionError) { "BackgroundCallCompaction:0"}}); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "BackgroundCallCompaction:0", [&](void*) { - fault_fs->SetFilesystemActive(false, - IOStatus::Corruption("Corruption")); + fault_fs_->SetFilesystemActive(false, + IOStatus::Corruption("Corruption")); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -806,20 +765,21 @@ TEST_F(DBErrorHandlingFSTest, CorruptionError) { ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_NE(s, Status::OK()); Destroy(options); } TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); Status s; @@ -829,13 +789,13 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { ASSERT_OK(Put(Key(0), "val")); SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ASSERT_EQ(listener->WaitForRecovery(5000000), true); s = Put(Key(1), "val"); @@ -848,13 +808,10 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { } TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); Status s; @@ -864,7 +821,7 @@ TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) { ASSERT_OK(Put(Key(0), "val")); SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); @@ -876,13 +833,14 @@ TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) { } TEST_F(DBErrorHandlingFSTest, WALWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.writable_file_max_buffer_size = 32768; options.listeners.emplace_back(listener); @@ -916,8 +874,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { write_error++; if (write_error > 2) { - fault_fs->SetFilesystemActive(false, - IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); } }); SyncPoint::GetInstance()->EnableProcessing(); @@ -927,7 +885,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { ASSERT_EQ(s, s.NoSpace()); } SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ASSERT_EQ(listener->WaitForRecovery(5000000), true); for (auto i = 0; i < 199; ++i) { if (i < 100) { @@ -948,13 +906,10 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { } TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.writable_file_max_buffer_size = 32768; options.listeners.emplace_back(listener); @@ -995,7 +950,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { write_error++; if (write_error > 2) { - fault_fs->SetFilesystemActive(false, error_msg); + fault_fs_->SetFilesystemActive(false, error_msg); } }); SyncPoint::GetInstance()->EnableProcessing(); @@ -1004,7 +959,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { s = dbfull()->Write(wopts, &batch); ASSERT_EQ(true, s.IsIOError()); } - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -1044,13 +999,14 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.writable_file_max_buffer_size = 32768; options.listeners.emplace_back(listener); @@ -1087,8 +1043,8 @@ TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { write_error++; if (write_error > 2) { - fault_fs->SetFilesystemActive(false, - IOStatus::NoSpace("Out of space")); + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); } }); SyncPoint::GetInstance()->EnableProcessing(); @@ -1098,7 +1054,7 @@ TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { ASSERT_EQ(s, s.NoSpace()); } SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ASSERT_EQ(listener->WaitForRecovery(5000000), true); for (auto i = 1; i < 4; ++i) { @@ -1129,7 +1085,11 @@ TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { } TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) { - FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(Env::Default()); + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(env_); std::vector> fault_envs; std::vector fault_fs; std::vector options; @@ -1142,7 +1102,7 @@ TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) { for (auto i = 0; i < kNumDbInstances; ++i) { listener.emplace_back(new ErrorHandlerFSListener()); options.emplace_back(GetDefaultOptions()); - fault_fs.emplace_back(new FaultInjectionTestFS(FileSystem::Default())); + fault_fs.emplace_back(new FaultInjectionTestFS(env_->GetFileSystem())); std::shared_ptr fs(fault_fs.back()); fault_envs.emplace_back(new CompositeEnvWrapper(def_env, fs)); options[i].env = fault_envs.back().get(); @@ -1230,7 +1190,11 @@ TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) { } TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) { - FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(Env::Default()); + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(env_); std::vector> fault_envs; std::vector fault_fs; std::vector options; @@ -1243,7 +1207,7 @@ TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) { for (auto i = 0; i < kNumDbInstances; ++i) { listener.emplace_back(new ErrorHandlerFSListener()); options.emplace_back(GetDefaultOptions()); - fault_fs.emplace_back(new FaultInjectionTestFS(FileSystem::Default())); + fault_fs.emplace_back(new FaultInjectionTestFS(env_->GetFileSystem())); std::shared_ptr fs(fault_fs.back()); fault_envs.emplace_back(new CompositeEnvWrapper(def_env, fs)); options[i].env = fault_envs.back().get(); @@ -1366,13 +1330,10 @@ TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) { // the bg error is cleaned unless the memtable is full. TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover1) { // Activate the FS before the first resume - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1393,7 +1354,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover1) { "FLushWritNoWALRetryableeErrorAutoRecover1:1"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); @@ -1403,7 +1364,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover1) { ASSERT_EQ("val1", Get(Key(1))); ASSERT_EQ("val1", Get(Key(1))); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ASSERT_OK(Put(Key(2), "val2", wo)); s = Flush(); // Since auto resume fails, the bg error is not cleand, flush will @@ -1424,13 +1385,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover1) { TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover2) { // Activate the FS before the first resume - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1448,14 +1406,14 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover2) { ASSERT_OK(Put(Key(1), "val1", wo)); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ("val1", Get(Key(1))); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ASSERT_EQ(listener->WaitForRecovery(5000000), true); ASSERT_EQ("val1", Get(Key(1))); ASSERT_OK(Put(Key(2), "val2", wo)); @@ -1469,13 +1427,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableeErrorAutoRecover2) { TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover1) { // Fail the first resume and make the second resume successful - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1498,12 +1453,12 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover1) { "FLushWritRetryableeErrorAutoRecover1:2"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover1:0"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover1:1"); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover1:2"); @@ -1522,13 +1477,10 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover1) { TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover2) { // Activate the FS before the first resume - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1544,13 +1496,13 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover2) { ASSERT_OK(Put(Key(1), "val1")); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ASSERT_EQ(listener->WaitForRecovery(5000000), true); ASSERT_EQ("val1", Get(Key(1))); @@ -1566,13 +1518,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover2) { TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover3) { // Fail all the resume and let user to resume - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1593,13 +1542,13 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover3) { "FLushWritRetryableeErrorAutoRecover3:1"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover3:0"); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover3:1"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -1620,13 +1569,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover3) { TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover4) { // Fail the first resume and does not do resume second time because // the IO error severity is Fatal Error and not Retryable. - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1649,10 +1595,10 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover4) { "RecoverFromRetryableBGIOError:RecoverFail0"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->SetCallBack( "RecoverFromRetryableBGIOError:BeforeResume1", - [&](void*) { fault_fs->SetFilesystemActive(false, nr_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, nr_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); @@ -1661,7 +1607,7 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover4) { TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover4:2"); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); // Even the FS is recoverd, due to the Fatal Error in bg_error_ the resume // and flush will all fail. ASSERT_EQ("val1", Get(Key(1))); @@ -1687,13 +1633,10 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover5) { // During the resume, call DB->CLose, make sure the resume thread exist // before close continues. Due to the shutdown, the resume is not successful // and the FS does not become active, so close status is still IO error - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1712,7 +1655,7 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover5) { "FLushWritRetryableeErrorAutoRecover5:0"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); @@ -1723,7 +1666,7 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_FLushWritRetryableeErrorAutoRecover5) { ASSERT_NE(s, Status::OK()); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); Reopen(options); ASSERT_NE("val1", Get(Key(1))); @@ -1739,13 +1682,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover6) { // During the resume, call DB->CLose, make sure the resume thread exist // before close continues. Due to the shutdown, the resume is not successful // and the FS does not become active, so close status is still IO error - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1770,13 +1710,13 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover6) { "FLushWritRetryableeErrorAutoRecover6:3"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover6:0"); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover6:1"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover6:2"); TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover6:3"); @@ -1798,13 +1738,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableeErrorAutoRecover6) { TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { // Fail the first resume and let the second resume be successful - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.max_bgerror_resume_count = 2; @@ -1832,12 +1769,12 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { "ManifestWriteRetryableErrorAutoRecover:2"}}); SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:0"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:1"); TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:2"); @@ -1854,13 +1791,10 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableErrorAutoRecover) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -1909,7 +1843,7 @@ TEST_F(DBErrorHandlingFSTest, ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", [&](void*) { if (fail_manifest.load()) { - fault_fs->SetFilesystemActive(false, error_msg); + fault_fs_->SetFilesystemActive(false, error_msg); } }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -1925,7 +1859,7 @@ TEST_F(DBErrorHandlingFSTest, ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); TEST_SYNC_POINT("CompactionManifestWriteErrorAR:2"); TEST_SYNC_POINT("CompactionManifestWriteErrorAR:3"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("CompactionManifestWriteErrorAR:4"); TEST_SYNC_POINT("CompactionManifestWriteErrorAR:5"); @@ -1948,13 +1882,10 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { // compaction, the FS is set to active and compaction is successful, so // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync // point. - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -1980,13 +1911,13 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { "CompactionWriteRetryableErrorAutoRecover0"}}); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "DBImpl::BackgroundCompaction:Start", - [&](void*) { fault_fs->SetFilesystemActive(true); }); + [&](void*) { fault_fs_->SetFilesystemActive(true); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "BackgroundCallCompaction:0", [&](void*) { fail_first.store(true); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "CompactionJob::OpenCompactionOutputFile", [&](void*) { if (fail_first.load() && fail_second.load()) { - fault_fs->SetFilesystemActive(false, error_msg); + fault_fs_->SetFilesystemActive(false, error_msg); fail_second.store(false); } }); @@ -2006,13 +1937,10 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { } TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.writable_file_max_buffer_size = 32768; options.listeners.emplace_back(listener); @@ -2058,7 +1986,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { write_error++; if (write_error > 2) { - fault_fs->SetFilesystemActive(false, error_msg); + fault_fs_->SetFilesystemActive(false, error_msg); } }); SyncPoint::GetInstance()->EnableProcessing(); @@ -2068,7 +1996,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { ASSERT_EQ(true, s.IsIOError()); TEST_SYNC_POINT("WALWriteError1:0"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("WALWriteError1:1"); TEST_SYNC_POINT("WALWriteError1:2"); @@ -2110,13 +2038,10 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { // Fail the first recover and try second time. - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.writable_file_max_buffer_size = 32768; options.listeners.emplace_back(listener); @@ -2162,7 +2087,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { write_error++; if (write_error > 2) { - fault_fs->SetFilesystemActive(false, error_msg); + fault_fs_->SetFilesystemActive(false, error_msg); } }); SyncPoint::GetInstance()->EnableProcessing(); @@ -2172,7 +2097,7 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { ASSERT_EQ(true, s.IsIOError()); TEST_SYNC_POINT("WALWriteError2:0"); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("WALWriteError2:1"); TEST_SYNC_POINT("WALWriteError2:2"); @@ -2216,13 +2141,10 @@ class DBErrorHandlingFencingTest : public DBErrorHandlingFSTest, public testing::WithParamInterface {}; TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.paranoid_checks = GetParam(); @@ -2233,27 +2155,24 @@ TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { ASSERT_OK(Put(Key(0), "val")); SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); + fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); ASSERT_TRUE(s.IsIOFenced()); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_TRUE(s.IsIOFenced()); Destroy(options); } TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.listeners.emplace_back(listener); options.paranoid_checks = GetParam(); @@ -2270,7 +2189,7 @@ TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { ASSERT_OK(Put(Key(1), "val")); SyncPoint::GetInstance()->SetCallBack( "VersionSet::LogAndApply:WriteManifest", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); + fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); @@ -2278,20 +2197,17 @@ TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { ASSERT_TRUE(s.IsIOFenced()); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_TRUE(s.IsIOFenced()); Close(); } TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.level0_file_num_compaction_trigger = 2; options.listeners.emplace_back(listener); @@ -2310,7 +2226,7 @@ TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { "BackgroundCallCompaction:0"}}); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "BackgroundCallCompaction:0", [&](void*) { - fault_fs->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); + fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -2322,20 +2238,17 @@ TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); ASSERT_TRUE(s.IsIOFenced()); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_TRUE(s.IsIOFenced()); Destroy(options); } TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) { - std::shared_ptr fault_fs( - new FaultInjectionTestFS(FileSystem::Default())); - std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); std::shared_ptr listener( new ErrorHandlerFSListener()); Options options = GetDefaultOptions(); - options.env = fault_fs_env.get(); + options.env = fault_env_.get(); options.create_if_missing = true; options.writable_file_max_buffer_size = 32768; options.listeners.emplace_back(listener); @@ -2370,8 +2283,8 @@ TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) { "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { write_error++; if (write_error > 2) { - fault_fs->SetFilesystemActive(false, - IOStatus::IOFenced("IO fenced")); + fault_fs_->SetFilesystemActive(false, + IOStatus::IOFenced("IO fenced")); } }); SyncPoint::GetInstance()->EnableProcessing(); @@ -2381,7 +2294,7 @@ TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) { ASSERT_TRUE(s.IsIOFenced()); } SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); + fault_fs_->SetFilesystemActive(true); { WriteBatch batch; diff --git a/db/external_sst_file_basic_test.cc b/db/external_sst_file_basic_test.cc index 624952563f..9f6d7cc325 100644 --- a/db/external_sst_file_basic_test.cc +++ b/db/external_sst_file_basic_test.cc @@ -24,7 +24,7 @@ class ExternalSSTFileBasicTest ExternalSSTFileBasicTest() : DBTestBase("/external_sst_file_basic_test", /*env_do_fsync=*/true) { sst_files_dir_ = dbname_ + "/sst_files/"; - fault_injection_test_env_.reset(new FaultInjectionTestEnv(Env::Default())); + fault_injection_test_env_.reset(new FaultInjectionTestEnv(env_)); DestroyAndRecreateExternalSSTFilesDir(); } @@ -1109,6 +1109,7 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) { } Options sst_file_writer_options; + sst_file_writer_options.env = env_; std::unique_ptr sst_file_writer( new SstFileWriter(EnvOptions(), sst_file_writer_options)); std::string file_name = @@ -1137,11 +1138,12 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) { TEST_F(ExternalSSTFileBasicTest, VerifyChecksumReadahead) { Options options; options.create_if_missing = true; - SpecialEnv senv(Env::Default()); + SpecialEnv senv(env_); options.env = &senv; DestroyAndReopen(options); Options sst_file_writer_options; + sst_file_writer_options.env = env_; std::unique_ptr sst_file_writer( new SstFileWriter(EnvOptions(), sst_file_writer_options)); std::string file_name = sst_files_dir_ + "verify_checksum_readahead_test.sst"; diff --git a/db/import_column_family_test.cc b/db/import_column_family_test.cc index fd77e04c5d..14b4847d09 100644 --- a/db/import_column_family_test.cc +++ b/db/import_column_family_test.cc @@ -16,8 +16,8 @@ class ImportColumnFamilyTest : public DBTestBase { ImportColumnFamilyTest() : DBTestBase("/import_column_family_test", /*env_do_fsync=*/true) { sst_files_dir_ = dbname_ + "/sst_files/"; - DestroyAndRecreateExternalSSTFilesDir(); export_files_dir_ = test::PerThreadDBPath(env_, "export"); + DestroyAndRecreateExternalSSTFilesDir(); import_cfh_ = nullptr; import_cfh2_ = nullptr; metadata_ptr_ = nullptr; diff --git a/db/listener_test.cc b/db/listener_test.cc index 5f0511d78f..92f879a132 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -676,7 +676,7 @@ class TableFileCreationListener : public EventListener { public: class TestEnv : public EnvWrapper { public: - TestEnv() : EnvWrapper(Env::Default()) {} + explicit TestEnv(Env* t) : EnvWrapper(t) {} void SetStatus(Status s) { status_ = s; } @@ -688,7 +688,7 @@ class TableFileCreationListener : public EventListener { return status_; } } - return Env::Default()->NewWritableFile(fname, result, options); + return target()->NewWritableFile(fname, result, options); } private: @@ -766,7 +766,6 @@ class TableFileCreationListener : public EventListener { } } - TestEnv test_env; int started_[2]; int finished_[2]; int failure_[2]; @@ -775,9 +774,11 @@ class TableFileCreationListener : public EventListener { TEST_F(EventListenerTest, TableFileCreationListenersTest) { auto listener = std::make_shared(); Options options; + std::unique_ptr test_env( + new TableFileCreationListener::TestEnv(CurrentOptions().env)); options.create_if_missing = true; options.listeners.push_back(listener); - options.env = &listener->test_env; + options.env = test_env.get(); DestroyAndReopen(options); ASSERT_OK(Put("foo", "aaa")); @@ -785,13 +786,12 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) { ASSERT_OK(Flush()); dbfull()->TEST_WaitForFlushMemTable(); listener->CheckAndResetCounters(1, 1, 0, 0, 0, 0); - ASSERT_OK(Put("foo", "aaa1")); ASSERT_OK(Put("bar", "bbb1")); - listener->test_env.SetStatus(Status::NotSupported("not supported")); + test_env->SetStatus(Status::NotSupported("not supported")); ASSERT_NOK(Flush()); listener->CheckAndResetCounters(1, 1, 1, 0, 0, 0); - listener->test_env.SetStatus(Status::OK()); + test_env->SetStatus(Status::OK()); Reopen(options); ASSERT_OK(Put("foo", "aaa2")); @@ -809,10 +809,11 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) { ASSERT_OK(Put("foo", "aaa3")); ASSERT_OK(Put("bar", "bbb3")); ASSERT_OK(Flush()); - listener->test_env.SetStatus(Status::NotSupported("not supported")); + test_env->SetStatus(Status::NotSupported("not supported")); dbfull()->CompactRange(CompactRangeOptions(), &kRangeStart, &kRangeEnd); dbfull()->TEST_WaitForCompact(); listener->CheckAndResetCounters(1, 1, 0, 1, 1, 1); + Close(); } class MemTableSealedListener : public EventListener { @@ -833,6 +834,7 @@ public: TEST_F(EventListenerTest, MemTableSealedListenerTest) { auto listener = std::make_shared(); Options options; + options.env = CurrentOptions().env; options.create_if_missing = true; options.listeners.push_back(listener); DestroyAndReopen(options); @@ -1066,7 +1068,7 @@ TEST_F(EventListenerTest, OnFileOperationTest) { TestFileOperationListener* listener = new TestFileOperationListener(); options.listeners.emplace_back(listener); - options.use_direct_io_for_flush_and_compaction = true; + options.use_direct_io_for_flush_and_compaction = false; Status s = TryReopen(options); if (s.IsInvalidArgument()) { options.use_direct_io_for_flush_and_compaction = false; diff --git a/db/perf_context_test.cc b/db/perf_context_test.cc index 5a714b9b85..7713f735e7 100644 --- a/db/perf_context_test.cc +++ b/db/perf_context_test.cc @@ -818,9 +818,7 @@ TEST_F(PerfContextTest, PerfContextByLevelGetSet) { TEST_F(PerfContextTest, CPUTimer) { if (Env::Default()->NowCPUNanos() == 0) { - // TODO: This should be a GTEST_SKIP when the embedded gtest is updated - // to 1.10 or higher. - GTEST_SUCCESS_("Skipped on target without NowCPUNanos support"); + ROCKSDB_GTEST_SKIP("Target without NowCPUNanos support"); return; } diff --git a/db/periodic_work_scheduler_test.cc b/db/periodic_work_scheduler_test.cc index d53265389d..9a18442870 100644 --- a/db/periodic_work_scheduler_test.cc +++ b/db/periodic_work_scheduler_test.cc @@ -13,8 +13,9 @@ namespace ROCKSDB_NAMESPACE { class PeriodicWorkSchedulerTest : public DBTestBase { public: PeriodicWorkSchedulerTest() - : DBTestBase("/periodic_work_scheduler_test", /*env_do_fsync=*/true), - mock_env_(new MockTimeEnv(Env::Default())) {} + : DBTestBase("/periodic_work_scheduler_test", /*env_do_fsync=*/true) { + mock_env_.reset(new MockTimeEnv(env_)); + } protected: std::unique_ptr mock_env_; diff --git a/db/version_set_test.cc b/db/version_set_test.cc index d912167513..57eae6f369 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -8,10 +8,11 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/version_set.h" + #include "db/db_impl/db_impl.h" #include "db/log_writer.h" -#include "env/mock_env.h" #include "logging/logging.h" +#include "table/block_based/block_based_table_factory.h" #include "table/mock_table.h" #include "test_util/testharness.h" #include "test_util/testutil.h" @@ -692,10 +693,7 @@ class VersionSetTestBase { int num_initial_edits_; explicit VersionSetTestBase(const std::string& name) - : mem_env_(nullptr), - env_(nullptr), - env_guard_(), - fs_(), + : env_(nullptr), dbname_(test::PerThreadDBPath(name)), options_(), db_options_(options_), @@ -707,25 +705,26 @@ class VersionSetTestBase { shutting_down_(false), mock_table_factory_(std::make_shared()) { const char* test_env_uri = getenv("TEST_ENV_URI"); - Env* base_env = nullptr; if (test_env_uri) { - Status s = Env::LoadEnv(test_env_uri, &base_env, &env_guard_); + Status s = Env::LoadEnv(test_env_uri, &env_, &env_guard_); EXPECT_OK(s); - EXPECT_NE(Env::Default(), base_env); + } else if (getenv("MEM_ENV")) { + env_guard_.reset(NewMemEnv(Env::Default())); + env_ = env_guard_.get(); } else { - base_env = Env::Default(); + env_ = Env::Default(); } - EXPECT_NE(nullptr, base_env); - if (getenv("MEM_ENV")) { - mem_env_ = new MockEnv(base_env); - } - env_ = mem_env_ ? mem_env_ : base_env; + EXPECT_NE(nullptr, env_); - fs_ = std::make_shared(env_); - EXPECT_OK(env_->CreateDirIfMissing(dbname_)); + fs_ = env_->GetFileSystem(); + EXPECT_OK(fs_->CreateDirIfMissing(dbname_, IOOptions(), nullptr)); + options_.env = env_; db_options_.env = env_; db_options_.fs = fs_; + immutable_cf_options_.env = env_; + immutable_cf_options_.fs = fs_.get(); + versions_.reset( new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), &write_buffer_manager_, &write_controller_, @@ -745,10 +744,6 @@ class VersionSetTestBase { options.env = env_; EXPECT_OK(DestroyDB(dbname_, options)); } - if (mem_env_) { - delete mem_env_; - mem_env_ = nullptr; - } } protected: @@ -760,7 +755,9 @@ class VersionSetTestBase { assert(log_writer != nullptr); VersionEdit new_db; if (db_options_.write_dbid_to_manifest) { - std::unique_ptr impl(new DBImpl(DBOptions(), dbname_)); + DBOptions tmp_db_options; + tmp_db_options.env = env_; + std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); std::string db_id; impl->GetDbIdentityFromIdentityFile(&db_id); new_db.SetDBId(db_id); @@ -873,7 +870,7 @@ class VersionSetTestBase { mutex_.Unlock(); } - MockEnv* mem_env_; + Env* mem_env_; Env* env_; std::shared_ptr env_guard_; std::shared_ptr fs_; @@ -2216,7 +2213,9 @@ class VersionSetTestEmptyDb assert(nullptr != log_writer); VersionEdit new_db; if (db_options_.write_dbid_to_manifest) { - std::unique_ptr impl(new DBImpl(DBOptions(), dbname_)); + DBOptions tmp_db_options; + tmp_db_options.env = env_; + std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); std::string db_id; impl->GetDbIdentityFromIdentityFile(&db_id); new_db.SetDBId(db_id); @@ -2541,7 +2540,9 @@ class VersionSetTestMissingFiles : public VersionSetTestBase, log_writer->reset(new log::Writer(std::move(file_writer), 0, false)); VersionEdit new_db; if (db_options_.write_dbid_to_manifest) { - std::unique_ptr impl(new DBImpl(DBOptions(), dbname_)); + DBOptions tmp_db_options; + tmp_db_options.env = env_; + std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); std::string db_id; impl->GetDbIdentityFromIdentityFile(&db_id); new_db.SetDBId(db_id); diff --git a/env/env_basic_test.cc b/env/env_basic_test.cc index cc91e10eb9..4099c8f2aa 100644 --- a/env/env_basic_test.cc +++ b/env/env_basic_test.cc @@ -228,8 +228,8 @@ TEST_P(EnvBasicTestWithParam, Basics) { ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/g")); ASSERT_OK(env_->GetChildren(test_dir_, &children)); ASSERT_EQ(0U, children.size()); - ASSERT_TRUE( - env_->GetChildren(test_dir_ + "/non_existent", &children).IsNotFound()); + Status s = env_->GetChildren(test_dir_ + "/non_existent", &children); + ASSERT_TRUE(s.IsNotFound()); } TEST_P(EnvBasicTestWithParam, ReadWrite) { diff --git a/env/mock_env.cc b/env/mock_env.cc index 3fdeac2b9e..d24557e5d1 100644 --- a/env/mock_env.cc +++ b/env/mock_env.cc @@ -8,10 +8,13 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "env/mock_env.h" + #include #include + #include "file/filename.h" #include "port/sys_time.h" +#include "rocksdb/file_system.h" #include "util/cast_util.h" #include "util/murmurhash.h" #include "util/random.h" @@ -78,7 +81,8 @@ class MemFile { uint64_t Size() const { return size_; } - void Truncate(size_t size) { + void Truncate(size_t size, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { MutexLock lock(&mutex_); if (size < size_) { data_.resize(size); @@ -100,7 +104,8 @@ class MemFile { } } - Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { + IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/, + Slice* result, char* scratch, IODebugContext* /*dbg*/) const { MutexLock lock(&mutex_); const uint64_t available = Size() - std::min(Size(), offset); size_t offset_ = static_cast(offset); @@ -109,7 +114,7 @@ class MemFile { } if (n == 0) { *result = Slice(); - return Status::OK(); + return IOStatus::OK(); } if (scratch) { memcpy(scratch, &(data_[offset_]), n); @@ -117,10 +122,11 @@ class MemFile { } else { *result = Slice(&(data_[offset_]), n); } - return Status::OK(); + return IOStatus::OK(); } - Status Write(uint64_t offset, const Slice& data) { + IOStatus Write(uint64_t offset, const Slice& data, + const IOOptions& /*options*/, IODebugContext* /*dbg*/) { MutexLock lock(&mutex_); size_t offset_ = static_cast(offset); if (offset + data.size() > data_.size()) { @@ -129,20 +135,21 @@ class MemFile { data_.replace(offset_, data.size(), data.data(), data.size()); size_ = data_.size(); modified_time_ = Now(); - return Status::OK(); + return IOStatus::OK(); } - Status Append(const Slice& data) { + IOStatus Append(const Slice& data, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { MutexLock lock(&mutex_); data_.append(data.data(), data.size()); size_ = data_.size(); modified_time_ = Now(); - return Status::OK(); + return IOStatus::OK(); } - Status Fsync() { + IOStatus Fsync(const IOOptions& /*options*/, IODebugContext* /*dbg*/) { fsynced_bytes_ = size_.load(); - return Status::OK(); + return IOStatus::OK(); } uint64_t ModifiedTime() const { return modified_time_; } @@ -177,111 +184,176 @@ class MemFile { namespace { -class MockSequentialFile : public SequentialFile { +class MockSequentialFile : public FSSequentialFile { public: - explicit MockSequentialFile(MemFile* file) : file_(file), pos_(0) { + explicit MockSequentialFile(MemFile* file, const FileOptions& opts) + : file_(file), + use_direct_io_(opts.use_direct_reads), + use_mmap_read_(opts.use_mmap_reads), + pos_(0) { file_->Ref(); } ~MockSequentialFile() override { file_->Unref(); } - Status Read(size_t n, Slice* result, char* scratch) override { - Status s = file_->Read(pos_, n, result, scratch); + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override { + IOStatus s = file_->Read(pos_, n, options, result, + (use_mmap_read_) ? nullptr : scratch, dbg); if (s.ok()) { pos_ += result->size(); } return s; } - Status Skip(uint64_t n) override { + bool use_direct_io() const override { return use_direct_io_; } + IOStatus Skip(uint64_t n) override { if (pos_ > file_->Size()) { - return Status::IOError("pos_ > file_->Size()"); + return IOStatus::IOError("pos_ > file_->Size()"); } const uint64_t available = file_->Size() - pos_; if (n > available) { n = available; } pos_ += static_cast(n); - return Status::OK(); + return IOStatus::OK(); } private: MemFile* file_; + bool use_direct_io_; + bool use_mmap_read_; size_t pos_; }; -class MockRandomAccessFile : public RandomAccessFile { +class MockRandomAccessFile : public FSRandomAccessFile { public: - explicit MockRandomAccessFile(MemFile* file) : file_(file) { file_->Ref(); } + explicit MockRandomAccessFile(MemFile* file, const FileOptions& opts) + : file_(file), + use_direct_io_(opts.use_direct_reads), + use_mmap_read_(opts.use_mmap_reads) { + file_->Ref(); + } ~MockRandomAccessFile() override { file_->Unref(); } - Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const override { - return file_->Read(offset, n, result, scratch); + bool use_direct_io() const override { return use_direct_io_; } + + IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + if (use_mmap_read_) { + return file_->Read(offset, n, options, result, nullptr, dbg); + } else { + return file_->Read(offset, n, options, result, scratch, dbg); + } } private: MemFile* file_; + bool use_direct_io_; + bool use_mmap_read_; }; -class MockRandomRWFile : public RandomRWFile { +class MockRandomRWFile : public FSRandomRWFile { public: explicit MockRandomRWFile(MemFile* file) : file_(file) { file_->Ref(); } ~MockRandomRWFile() override { file_->Unref(); } - Status Write(uint64_t offset, const Slice& data) override { - return file_->Write(offset, data); + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { + return file_->Write(offset, data, options, dbg); } - Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const override { - return file_->Read(offset, n, result, scratch); + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + return file_->Read(offset, n, options, result, scratch, dbg); } - Status Close() override { return file_->Fsync(); } + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } - Status Flush() override { return Status::OK(); } + IOStatus Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } - Status Sync() override { return file_->Fsync(); } + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } private: MemFile* file_; }; -class MockWritableFile : public WritableFile { +class MockWritableFile : public FSWritableFile { public: - MockWritableFile(MemFile* file, RateLimiter* rate_limiter) - : file_(file), rate_limiter_(rate_limiter) { + MockWritableFile(MemFile* file, const FileOptions& opts) + : file_(file), + use_direct_io_(opts.use_direct_writes), + rate_limiter_(opts.rate_limiter) { file_->Ref(); } ~MockWritableFile() override { file_->Unref(); } - Status Append(const Slice& data) override { + bool use_direct_io() const override { return false && use_direct_io_; } + + using FSWritableFile::Append; + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { size_t bytes_written = 0; while (bytes_written < data.size()) { auto bytes = RequestToken(data.size() - bytes_written); - Status s = file_->Append(Slice(data.data() + bytes_written, bytes)); + IOStatus s = file_->Append(Slice(data.data() + bytes_written, bytes), + options, dbg); if (!s.ok()) { return s; } bytes_written += bytes; } - return Status::OK(); + return IOStatus::OK(); } - Status Truncate(uint64_t size) override { - file_->Truncate(static_cast(size)); - return Status::OK(); + + using FSWritableFile::PositionedAppend; + IOStatus PositionedAppend(const Slice& data, uint64_t /*offset*/, + const IOOptions& options, + IODebugContext* dbg) override { + assert(use_direct_io_); + return Append(data, options, dbg); } - Status Close() override { return file_->Fsync(); } - Status Flush() override { return Status::OK(); } + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override { + file_->Truncate(static_cast(size), options, dbg); + return IOStatus::OK(); + } + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } - Status Sync() override { return file_->Fsync(); } + IOStatus Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } - uint64_t GetFileSize() override { return file_->Size(); } + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } + + uint64_t GetFileSize(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return file_->Size(); + } private: inline size_t RequestToken(size_t bytes) { @@ -294,12 +366,16 @@ class MockWritableFile : public WritableFile { } MemFile* file_; + bool use_direct_io_; RateLimiter* rate_limiter_; }; -class MockEnvDirectory : public Directory { +class MockEnvDirectory : public FSDirectory { public: - Status Fsync() override { return Status::OK(); } + IOStatus Fsync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } }; class MockEnvFileLock : public FileLock { @@ -314,21 +390,26 @@ class MockEnvFileLock : public FileLock { class TestMemLogger : public Logger { private: - std::unique_ptr file_; + std::unique_ptr file_; std::atomic_size_t log_size_; static const uint64_t flush_every_seconds_ = 5; std::atomic_uint_fast64_t last_flush_micros_; Env* env_; + IOOptions options_; + IODebugContext* dbg_; std::atomic flush_pending_; public: - TestMemLogger(std::unique_ptr f, Env* env, + TestMemLogger(std::unique_ptr f, Env* env, + const IOOptions& options, IODebugContext* dbg, const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL) : Logger(log_level), file_(std::move(f)), log_size_(0), last_flush_micros_(0), env_(env), + options_(options), + dbg_(dbg), flush_pending_(false) {} ~TestMemLogger() override {} @@ -394,7 +475,7 @@ class TestMemLogger : public Logger { assert(p <= limit); const size_t write_size = p - base; - Status s = file_->Append(Slice(base, write_size)); + Status s = file_->Append(Slice(base, write_size), options_, dbg_); if (s.ok()) { flush_pending_ = true; log_size_ += write_size; @@ -414,151 +495,304 @@ class TestMemLogger : public Logger { size_t GetLogFileSize() const override { return log_size_; } }; +class MockFileSystem : public FileSystem { + public: + explicit MockFileSystem(Env* env, bool supports_direct_io = true) + : env_(env), supports_direct_io_(supports_direct_io) {} + + ~MockFileSystem() override { + for (auto i = file_map_.begin(); i != file_map_.end(); ++i) { + i->second->Unref(); + } + } + + const char* Name() const override { return "Memory"; } + IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + IOStatus NewRandomAccessFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewDirectory(const std::string& /*name*/, const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus FileExists(const std::string& fname, const IOOptions& /*io_opts*/, + IODebugContext* /*dbg*/) override; + IOStatus GetChildren(const std::string& dir, const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Truncate(const std::string& fname, size_t size, + const IOOptions& options, IODebugContext* dbg) override; + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override; + IOStatus RenameFile(const std::string& src, const std::string& target, + const IOOptions& options, IODebugContext* dbg) override; + IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override; + IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) override; + IOStatus UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override; + IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) override; + // Get full directory name for this db. + IOStatus GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* /*dbg*/) override { + *output_path = NormalizeMockPath(db_path); + if (output_path->at(0) != '/') { + return IOStatus::NotSupported("GetAbsolutePath"); + } else { + return IOStatus::OK(); + } + } + IOStatus IsDirectory(const std::string& /*path*/, + const IOOptions& /*options*/, bool* /*is_dir*/, + IODebugContext* /*dgb*/) override { + return IOStatus::NotSupported("IsDirectory"); + } + + Status CorruptBuffer(const std::string& fname); + + private: + bool RenameFileInternal(const std::string& src, const std::string& dest); + void DeleteFileInternal(const std::string& fname); + bool GetChildrenInternal(const std::string& fname, + std::vector* results); + + std::string NormalizeMockPath(const std::string& path) { + std::string p = NormalizePath(path); + if (p.back() == '/' && p.size() > 1) { + p.pop_back(); + } + return p; + } + + private: + // Map from filenames to MemFile objects, representing a simple file system. + port::Mutex mutex_; + std::map file_map_; // Protected by mutex_. + Env* env_; + bool supports_direct_io_; +}; + } // Anonymous namespace - -MockEnv::MockEnv(Env* base_env) : EnvWrapper(base_env), fake_sleep_micros_(0) {} - -MockEnv::~MockEnv() { - for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i) { - i->second->Unref(); - } -} - // Partial implementation of the Env interface. -Status MockEnv::NewSequentialFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& /*soptions*/) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::NewSequentialFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); if (file_map_.find(fn) == file_map_.end()) { *result = nullptr; - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } auto* f = file_map_[fn]; if (f->is_lock_file()) { - return Status::InvalidArgument(fn, "Cannot open a lock file."); + return IOStatus::InvalidArgument(fn, "Cannot open a lock file."); + } else if (file_opts.use_direct_reads && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockSequentialFile(f, file_opts)); + return IOStatus::OK(); } - result->reset(new MockSequentialFile(f)); - return Status::OK(); } -Status MockEnv::NewRandomAccessFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& /*soptions*/) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::NewRandomAccessFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); if (file_map_.find(fn) == file_map_.end()) { *result = nullptr; - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } auto* f = file_map_[fn]; if (f->is_lock_file()) { - return Status::InvalidArgument(fn, "Cannot open a lock file."); + return IOStatus::InvalidArgument(fn, "Cannot open a lock file."); + } else if (file_opts.use_direct_reads && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockRandomAccessFile(f, file_opts)); + return IOStatus::OK(); } - result->reset(new MockRandomAccessFile(f)); - return Status::OK(); } -Status MockEnv::NewRandomRWFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& /*soptions*/) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::NewRandomRWFile( + const std::string& fname, const FileOptions& /*file_opts*/, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); if (file_map_.find(fn) == file_map_.end()) { *result = nullptr; - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } auto* f = file_map_[fn]; if (f->is_lock_file()) { - return Status::InvalidArgument(fn, "Cannot open a lock file."); + return IOStatus::InvalidArgument(fn, "Cannot open a lock file."); } result->reset(new MockRandomRWFile(f)); - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::ReuseWritableFile(const std::string& fname, - const std::string& old_fname, - std::unique_ptr* result, - const EnvOptions& options) { - auto s = RenameFile(old_fname, fname); +IOStatus MockFileSystem::ReuseWritableFile( + const std::string& fname, const std::string& old_fname, + const FileOptions& options, std::unique_ptr* result, + IODebugContext* dbg) { + auto s = RenameFile(old_fname, fname, IOOptions(), dbg); if (!s.ok()) { return s; + } else { + result->reset(); + return NewWritableFile(fname, options, result, dbg); } - result->reset(); - return NewWritableFile(fname, result, options); } -Status MockEnv::NewWritableFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& env_options) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::NewWritableFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); if (file_map_.find(fn) != file_map_.end()) { DeleteFileInternal(fn); } - MemFile* file = new MemFile(this, fn, false); + MemFile* file = new MemFile(env_, fn, false); file->Ref(); file_map_[fn] = file; - - result->reset(new MockWritableFile(file, env_options.rate_limiter)); - return Status::OK(); + if (file_opts.use_direct_writes && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockWritableFile(file, file_opts)); + return IOStatus::OK(); + } } -Status MockEnv::NewDirectory(const std::string& /*name*/, - std::unique_ptr* result) { +IOStatus MockFileSystem::ReopenWritableFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + MemFile* file = nullptr; + if (file_map_.find(fn) == file_map_.end()) { + file = new MemFile(env_, fn, false); + file_map_[fn] = file; + } else { + file = file_map_[fn]; + } + file->Ref(); + if (file_opts.use_direct_writes && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockWritableFile(file, file_opts)); + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::NewDirectory(const std::string& /*name*/, + const IOOptions& /*io_opts*/, + std::unique_ptr* result, + IODebugContext* /*dbg*/) { result->reset(new MockEnvDirectory()); - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::FileExists(const std::string& fname) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::FileExists(const std::string& fname, + const IOOptions& /*io_opts*/, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); if (file_map_.find(fn) != file_map_.end()) { // File exists - return Status::OK(); + return IOStatus::OK(); } // Now also check if fn exists as a dir for (const auto& iter : file_map_) { const std::string& filename = iter.first; if (filename.size() >= fn.size() + 1 && filename[fn.size()] == '/' && Slice(filename).starts_with(Slice(fn))) { - return Status::OK(); + return IOStatus::OK(); } } - return Status::NotFound(); + return IOStatus::NotFound(); } -Status MockEnv::GetChildren(const std::string& dir, - std::vector* result) { - auto d = NormalizePath(dir); +bool MockFileSystem::GetChildrenInternal(const std::string& dir, + std::vector* result) { + auto d = NormalizeMockPath(dir); bool found_dir = false; - { - MutexLock lock(&mutex_); - result->clear(); - for (const auto& iter : file_map_) { - const std::string& filename = iter.first; + result->clear(); + for (const auto& iter : file_map_) { + const std::string& filename = iter.first; - if (filename == d) { - found_dir = true; - } else if (filename.size() >= d.size() + 1 && filename[d.size()] == '/' && - Slice(filename).starts_with(Slice(d))) { - found_dir = true; - size_t next_slash = filename.find('/', d.size() + 1); - if (next_slash != std::string::npos) { - result->push_back( - filename.substr(d.size() + 1, next_slash - d.size() - 1)); - } else { - result->push_back(filename.substr(d.size() + 1)); - } + if (filename == d) { + found_dir = true; + } else if (filename.size() >= d.size() + 1 && filename[d.size()] == '/' && + Slice(filename).starts_with(Slice(d))) { + found_dir = true; + size_t next_slash = filename.find('/', d.size() + 1); + if (next_slash != std::string::npos) { + result->push_back( + filename.substr(d.size() + 1, next_slash - d.size() - 1)); + } else { + result->push_back(filename.substr(d.size() + 1)); } } } result->erase(std::unique(result->begin(), result->end()), result->end()); - return found_dir ? Status::OK() : Status::NotFound(); + return found_dir; } -void MockEnv::DeleteFileInternal(const std::string& fname) { - assert(fname == NormalizePath(fname)); +IOStatus MockFileSystem::GetChildren(const std::string& dir, + const IOOptions& /*options*/, + std::vector* result, + IODebugContext* /*dbg*/) { + MutexLock lock(&mutex_); + bool found_dir = GetChildrenInternal(dir, result); + return found_dir ? IOStatus::OK() : IOStatus::NotFound(dir); +} + +void MockFileSystem::DeleteFileInternal(const std::string& fname) { + assert(fname == NormalizeMockPath(fname)); const auto& pair = file_map_.find(fname); if (pair != file_map_.end()) { pair->second->Unref(); @@ -566,180 +800,222 @@ void MockEnv::DeleteFileInternal(const std::string& fname) { } } -Status MockEnv::DeleteFile(const std::string& fname) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::DeleteFile(const std::string& fname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); if (file_map_.find(fn) == file_map_.end()) { - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } DeleteFileInternal(fn); - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::Truncate(const std::string& fname, size_t size) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::Truncate(const std::string& fname, size_t size, + const IOOptions& options, + IODebugContext* dbg) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); auto iter = file_map_.find(fn); if (iter == file_map_.end()) { - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } - iter->second->Truncate(size); - return Status::OK(); + iter->second->Truncate(size, options, dbg); + return IOStatus::OK(); } -Status MockEnv::CreateDir(const std::string& dirname) { - auto dn = NormalizePath(dirname); +IOStatus MockFileSystem::CreateDir(const std::string& dirname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto dn = NormalizeMockPath(dirname); MutexLock lock(&mutex_); if (file_map_.find(dn) == file_map_.end()) { - MemFile* file = new MemFile(this, dn, false); + MemFile* file = new MemFile(env_, dn, false); file->Ref(); file_map_[dn] = file; } else { - return Status::IOError(); + return IOStatus::IOError(); } - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::CreateDirIfMissing(const std::string& dirname) { - CreateDir(dirname).PermitUncheckedError(); - return Status::OK(); +IOStatus MockFileSystem::CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + CreateDir(dirname, options, dbg).PermitUncheckedError(); + return IOStatus::OK(); } -Status MockEnv::DeleteDir(const std::string& dirname) { - return DeleteFile(dirname); +IOStatus MockFileSystem::DeleteDir(const std::string& dirname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto dir = NormalizeMockPath(dirname); + MutexLock lock(&mutex_); + if (file_map_.find(dir) == file_map_.end()) { + return IOStatus::PathNotFound(dir); + } else { + std::vector children; + if (GetChildrenInternal(dir, &children)) { + for (const auto& child : children) { + DeleteFileInternal(child); + } + } + DeleteFileInternal(dir); + return IOStatus::OK(); + } } -Status MockEnv::GetFileSize(const std::string& fname, uint64_t* file_size) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::GetFileSize(const std::string& fname, + const IOOptions& /*options*/, + uint64_t* file_size, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); auto iter = file_map_.find(fn); if (iter == file_map_.end()) { - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } *file_size = iter->second->Size(); - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::GetFileModificationTime(const std::string& fname, - uint64_t* time) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::GetFileModificationTime(const std::string& fname, + const IOOptions& /*options*/, + uint64_t* time, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); auto iter = file_map_.find(fn); if (iter == file_map_.end()) { - return Status::IOError(fn, "File not found"); + return IOStatus::PathNotFound(fn); } *time = iter->second->ModifiedTime(); - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::RenameFile(const std::string& src, const std::string& dest) { - auto s = NormalizePath(src); - auto t = NormalizePath(dest); - MutexLock lock(&mutex_); - if (file_map_.find(s) == file_map_.end()) { - return Status::IOError(s, "File not found"); +bool MockFileSystem::RenameFileInternal(const std::string& src, + const std::string& dest) { + if (file_map_.find(src) == file_map_.end()) { + return false; + } else { + std::vector children; + if (GetChildrenInternal(src, &children)) { + for (const auto& child : children) { + RenameFileInternal(src + "/" + child, dest + "/" + child); + } + } + DeleteFileInternal(dest); + file_map_[dest] = file_map_[src]; + file_map_.erase(src); + return true; } - - DeleteFileInternal(t); - file_map_[t] = file_map_[s]; - file_map_.erase(s); - return Status::OK(); } -Status MockEnv::LinkFile(const std::string& src, const std::string& dest) { - auto s = NormalizePath(src); - auto t = NormalizePath(dest); +IOStatus MockFileSystem::RenameFile(const std::string& src, + const std::string& dest, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto s = NormalizeMockPath(src); + auto t = NormalizeMockPath(dest); + MutexLock lock(&mutex_); + bool found = RenameFileInternal(s, t); + if (!found) { + return IOStatus::PathNotFound(s); + } else { + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::LinkFile(const std::string& src, + const std::string& dest, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto s = NormalizeMockPath(src); + auto t = NormalizeMockPath(dest); MutexLock lock(&mutex_); if (file_map_.find(s) == file_map_.end()) { - return Status::IOError(s, "File not found"); + return IOStatus::PathNotFound(s); } DeleteFileInternal(t); file_map_[t] = file_map_[s]; file_map_[t]->Ref(); // Otherwise it might get deleted when noone uses s - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::NewLogger(const std::string& fname, - std::shared_ptr* result) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::NewLogger(const std::string& fname, + const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); auto iter = file_map_.find(fn); MemFile* file = nullptr; if (iter == file_map_.end()) { - file = new MemFile(this, fn, false); + file = new MemFile(env_, fn, false); file->Ref(); file_map_[fn] = file; } else { file = iter->second; } - std::unique_ptr f(new MockWritableFile(file, nullptr)); - result->reset(new TestMemLogger(std::move(f), this)); - return Status::OK(); + std::unique_ptr f(new MockWritableFile(file, FileOptions())); + result->reset(new TestMemLogger(std::move(f), env_, io_opts, dbg)); + return IOStatus::OK(); } -Status MockEnv::LockFile(const std::string& fname, FileLock** flock) { - auto fn = NormalizePath(fname); +IOStatus MockFileSystem::LockFile(const std::string& fname, + const IOOptions& /*options*/, + FileLock** flock, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); { MutexLock lock(&mutex_); if (file_map_.find(fn) != file_map_.end()) { if (!file_map_[fn]->is_lock_file()) { - return Status::InvalidArgument(fname, "Not a lock file."); + return IOStatus::InvalidArgument(fname, "Not a lock file."); } if (!file_map_[fn]->Lock()) { - return Status::IOError(fn, "Lock is already held."); + return IOStatus::IOError(fn, "lock is already held."); } } else { - auto* file = new MemFile(this, fn, true); + auto* file = new MemFile(env_, fn, true); file->Ref(); file->Lock(); file_map_[fn] = file; } } *flock = new MockEnvFileLock(fn); - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::UnlockFile(FileLock* flock) { +IOStatus MockFileSystem::UnlockFile(FileLock* flock, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { std::string fn = static_cast_with_check(flock)->FileName(); { MutexLock lock(&mutex_); if (file_map_.find(fn) != file_map_.end()) { if (!file_map_[fn]->is_lock_file()) { - return Status::InvalidArgument(fn, "Not a lock file."); + return IOStatus::InvalidArgument(fn, "Not a lock file."); } file_map_[fn]->Unlock(); } } delete flock; - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::GetTestDirectory(std::string* path) { +IOStatus MockFileSystem::GetTestDirectory(const IOOptions& /*options*/, + std::string* path, + IODebugContext* /*dbg*/) { *path = "/test"; - return Status::OK(); + return IOStatus::OK(); } -Status MockEnv::GetCurrentTime(int64_t* unix_time) { - auto s = EnvWrapper::GetCurrentTime(unix_time); - if (s.ok()) { - *unix_time += fake_sleep_micros_.load() / (1000 * 1000); - } - return s; -} - -uint64_t MockEnv::NowMicros() { - return EnvWrapper::NowMicros() + fake_sleep_micros_.load(); -} - -uint64_t MockEnv::NowNanos() { - return EnvWrapper::NowNanos() + fake_sleep_micros_.load() * 1000; -} - -Status MockEnv::CorruptBuffer(const std::string& fname) { - auto fn = NormalizePath(fname); +Status MockFileSystem::CorruptBuffer(const std::string& fname) { + auto fn = NormalizeMockPath(fname); MutexLock lock(&mutex_); auto iter = file_map_.find(fn); if (iter == file_map_.end()) { @@ -749,6 +1025,31 @@ Status MockEnv::CorruptBuffer(const std::string& fname) { return Status::OK(); } +MockEnv::MockEnv(Env* base_env) + : CompositeEnvWrapper(base_env, std::make_shared(this)), + fake_sleep_micros_(0) {} + +Status MockEnv::GetCurrentTime(int64_t* unix_time) { + auto s = CompositeEnvWrapper::GetCurrentTime(unix_time); + if (s.ok()) { + *unix_time += fake_sleep_micros_.load() / (1000 * 1000); + } + return s; +} + +uint64_t MockEnv::NowMicros() { + return CompositeEnvWrapper::NowMicros() + fake_sleep_micros_.load(); +} + +uint64_t MockEnv::NowNanos() { + return CompositeEnvWrapper::NowNanos() + fake_sleep_micros_.load() * 1000; +} + +Status MockEnv::CorruptBuffer(const std::string& fname) { + auto mock = static_cast_with_check(GetFileSystem().get()); + return mock->CorruptBuffer(fname); +} + void MockEnv::FakeSleepForMicroseconds(int64_t micros) { fake_sleep_micros_.fetch_add(micros); } diff --git a/env/mock_env.h b/env/mock_env.h index 1ed5c0b1f7..24965849d9 100644 --- a/env/mock_env.h +++ b/env/mock_env.h @@ -12,88 +12,17 @@ #include #include #include + +#include "env/composite_env_wrapper.h" #include "rocksdb/env.h" #include "rocksdb/status.h" -#include "port/port.h" -#include "util/mutexlock.h" namespace ROCKSDB_NAMESPACE { -class MemFile; -class MockEnv : public EnvWrapper { +class MockEnv : public CompositeEnvWrapper { public: explicit MockEnv(Env* base_env); - ~MockEnv() override; - - // Partial implementation of the Env interface. - Status RegisterDbPaths(const std::vector& /*paths*/) override { - return Status::OK(); - } - - Status UnregisterDbPaths(const std::vector& /*paths*/) override { - return Status::OK(); - } - - Status NewSequentialFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& soptions) override; - - Status NewRandomAccessFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& soptions) override; - - Status NewRandomRWFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& options) override; - - Status ReuseWritableFile(const std::string& fname, - const std::string& old_fname, - std::unique_ptr* result, - const EnvOptions& options) override; - - Status NewWritableFile(const std::string& fname, - std::unique_ptr* result, - const EnvOptions& env_options) override; - - Status NewDirectory(const std::string& name, - std::unique_ptr* result) override; - - Status FileExists(const std::string& fname) override; - - Status GetChildren(const std::string& dir, - std::vector* result) override; - - void DeleteFileInternal(const std::string& fname); - - Status DeleteFile(const std::string& fname) override; - - Status Truncate(const std::string& fname, size_t size) override; - - Status CreateDir(const std::string& dirname) override; - - Status CreateDirIfMissing(const std::string& dirname) override; - - Status DeleteDir(const std::string& dirname) override; - - Status GetFileSize(const std::string& fname, uint64_t* file_size) override; - - Status GetFileModificationTime(const std::string& fname, - uint64_t* time) override; - - Status RenameFile(const std::string& src, const std::string& target) override; - - Status LinkFile(const std::string& src, const std::string& target) override; - - Status NewLogger(const std::string& fname, - std::shared_ptr* result) override; - - Status LockFile(const std::string& fname, FileLock** flock) override; - - Status UnlockFile(FileLock* flock) override; - - Status GetTestDirectory(std::string* path) override; - // Results of these can be affected by FakeSleepForMicroseconds() Status GetCurrentTime(int64_t* unix_time) override; uint64_t NowMicros() override; @@ -106,11 +35,6 @@ class MockEnv : public EnvWrapper { void FakeSleepForMicroseconds(int64_t micros); private: - // Map from filenames to MemFile objects, representing a simple file system. - typedef std::map FileSystem; - port::Mutex mutex_; - FileSystem file_map_; // Protected by mutex_. - std::atomic fake_sleep_micros_; }; diff --git a/file/file_util.cc b/file/file_util.cc index 647043f7e5..2d487dcc46 100644 --- a/file/file_util.cc +++ b/file/file_util.cc @@ -211,6 +211,8 @@ Status DestroyDir(Env* env, const std::string& dir) { } else { s = env->DeleteFile(path); } + } else if (s.IsNotSupported()) { + s = Status::OK(); } if (!s.ok()) { // IsDirectory, etc. might not report NotFound diff --git a/file/filename.cc b/file/filename.cc index ac48c76e00..9836971f4f 100644 --- a/file/filename.cc +++ b/file/filename.cc @@ -184,7 +184,8 @@ InfoLogPrefix::InfoLogPrefix(bool has_log_dir, snprintf(buf, sizeof(buf), kInfoLogPrefix); prefix = Slice(buf, sizeof(kInfoLogPrefix) - 1); } else { - size_t len = GetInfoLogPrefix(db_absolute_path, buf, sizeof(buf)); + size_t len = + GetInfoLogPrefix(NormalizePath(db_absolute_path), buf, sizeof(buf)); prefix = Slice(buf, len); } } diff --git a/file/prefetch_test.cc b/file/prefetch_test.cc index ffe0367a4b..8ebf5ea13a 100644 --- a/file/prefetch_test.cc +++ b/file/prefetch_test.cc @@ -25,7 +25,7 @@ class MockRandomAccessFile : public FSRandomAccessFileWrapper { prefetch_count_.fetch_add(1); return target()->Prefetch(offset, n, options, dbg); } else { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("Prefetch not supported"); } } @@ -37,9 +37,9 @@ class MockRandomAccessFile : public FSRandomAccessFileWrapper { class MockFS : public FileSystemWrapper { public: - explicit MockFS(bool support_prefetch) - : FileSystemWrapper(FileSystem::Default()), - support_prefetch_(support_prefetch) {} + explicit MockFS(const std::shared_ptr& wrapped, + bool support_prefetch) + : FileSystemWrapper(wrapped), support_prefetch_(support_prefetch) {} IOStatus NewRandomAccessFile(const std::string& fname, const FileOptions& opts, @@ -79,9 +79,9 @@ TEST_P(PrefetchTest, Basic) { // Second param is if directIO is enabled or not bool use_direct_io = std::get<1>(GetParam()); - const int kNumKeys = 1100; - std::shared_ptr fs = std::make_shared(support_prefetch); + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); Options options = CurrentOptions(); options.write_buffer_size = 1024; diff --git a/include/rocksdb/file_system.h b/include/rocksdb/file_system.h index 3683491c1b..e38929db60 100644 --- a/include/rocksdb/file_system.h +++ b/include/rocksdb/file_system.h @@ -262,7 +262,7 @@ class FileSystem { virtual IOStatus ReopenWritableFile( const std::string& /*fname*/, const FileOptions& /*options*/, std::unique_ptr* /*result*/, IODebugContext* /*dbg*/) { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("ReopenWritableFile"); } // Reuse an existing file by renaming it and opening it as writable. @@ -523,7 +523,7 @@ class FileSystem { const IOOptions& /*options*/, uint64_t* /*diskfree*/, IODebugContext* /*dbg*/) { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("GetFreeSpace"); } virtual IOStatus IsDirectory(const std::string& /*path*/, @@ -584,7 +584,7 @@ class FSSequentialFile { const IOOptions& /*options*/, Slice* /*result*/, char* /*scratch*/, IODebugContext* /*dbg*/) { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("PositionedRead"); } // If you're adding methods here, remember to add them to @@ -638,7 +638,7 @@ class FSRandomAccessFile { virtual IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/, const IOOptions& /*options*/, IODebugContext* /*dbg*/) { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("Prefetch"); } // Read a bunch of blocks as described by reqs. The blocks can @@ -770,7 +770,7 @@ class FSWritableFile { uint64_t /* offset */, const IOOptions& /*options*/, IODebugContext* /*dbg*/) { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("PositionedAppend"); } // EXPERIMENTAL / CURRENTLY UNUSED @@ -782,7 +782,7 @@ class FSWritableFile { const IOOptions& /*options*/, const DataVerificationInfo& /* verification_info */, IODebugContext* /*dbg*/) { - return IOStatus::NotSupported(); + return IOStatus::NotSupported("PositionedAppend"); } // Truncate is necessary to trim the file to the correct size diff --git a/monitoring/stats_history_test.cc b/monitoring/stats_history_test.cc index a1affb6d19..37e8f6d4ad 100644 --- a/monitoring/stats_history_test.cc +++ b/monitoring/stats_history_test.cc @@ -32,8 +32,9 @@ namespace ROCKSDB_NAMESPACE { class StatsHistoryTest : public DBTestBase { public: StatsHistoryTest() - : DBTestBase("/stats_history_test", /*env_do_fsync=*/true), - mock_env_(new MockTimeEnv(Env::Default())) {} + : DBTestBase("/stats_history_test", /*env_do_fsync=*/true) { + mock_env_.reset(new MockTimeEnv(env_)); + } protected: std::unique_ptr mock_env_; diff --git a/table/block_based/block_based_table_reader_test.cc b/table/block_based/block_based_table_reader_test.cc index be2af3195d..f714ce8682 100644 --- a/table/block_based/block_based_table_reader_test.cc +++ b/table/block_based/block_based_table_reader_test.cc @@ -299,7 +299,8 @@ TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) { table.reset(); // Corrupt the block pointed to by handle - test::CorruptFile(Path(table_name), static_cast(handle.offset()), 128); + ASSERT_OK(test::CorruptFile(options.env, Path(table_name), + static_cast(handle.offset()), 128)); NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table); Status s = table->VerifyChecksum(ReadOptions(), diff --git a/test_util/testharness.h b/test_util/testharness.h index 60a195e2b7..3dcde1b6b5 100644 --- a/test_util/testharness.h +++ b/test_util/testharness.h @@ -15,6 +15,13 @@ #include #endif +// If GTEST_SKIP is available, use it. Otherwise, define skip as success +#ifdef GTEST_SKIP_ +#define ROCKSDB_GTEST_SKIP(m) GTEST_SKIP_(m) +#else +#define ROCKSDB_GTEST_SKIP(m) GTEST_SUCCESS_("SKIPPED: " m) +#endif + #include #include "rocksdb/env.h" diff --git a/test_util/testutil.cc b/test_util/testutil.cc index 18349cf4c1..276629dd75 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -486,46 +486,62 @@ size_t GetLinesCount(const std::string& fname, const std::string& pattern) { return count; } - -void CorruptFile(const std::string& fname, int offset, int bytes_to_corrupt) { - struct stat sbuf; - if (stat(fname.c_str(), &sbuf) != 0) { - // strerror is not thread-safe so should not be used in the "passing" path - // of unit tests (sometimes parallelized) but is OK here where test fails - const char* msg = strerror(errno); - fprintf(stderr, "%s:%s\n", fname.c_str(), msg); - assert(false); - } - - if (offset < 0) { +Status CorruptFile(Env* env, const std::string& fname, int offset, + int bytes_to_corrupt, bool verify_checksum /*=true*/) { + uint64_t size; + Status s = env->GetFileSize(fname, &size); + if (!s.ok()) { + return s; + } else if (offset < 0) { // Relative to end of file; make it absolute - if (-offset > sbuf.st_size) { + if (-offset > static_cast(size)) { offset = 0; } else { - offset = static_cast(sbuf.st_size + offset); + offset = static_cast(size + offset); } } - if (offset > sbuf.st_size) { - offset = static_cast(sbuf.st_size); + if (offset > static_cast(size)) { + offset = static_cast(size); } - if (offset + bytes_to_corrupt > sbuf.st_size) { - bytes_to_corrupt = static_cast(sbuf.st_size - offset); + if (offset + bytes_to_corrupt > static_cast(size)) { + bytes_to_corrupt = static_cast(size - offset); } // Do it std::string contents; - Status s = ReadFileToString(Env::Default(), fname, &contents); - assert(s.ok()); - for (int i = 0; i < bytes_to_corrupt; i++) { - contents[i + offset] ^= 0x80; + s = ReadFileToString(env, fname, &contents); + if (s.ok()) { + for (int i = 0; i < bytes_to_corrupt; i++) { + contents[i + offset] ^= 0x80; + } + s = WriteStringToFile(env, contents, fname); } - s = WriteStringToFile(Env::Default(), contents, fname); - assert(s.ok()); - Options options; - EnvOptions env_options; + if (s.ok() && verify_checksum) { #ifndef ROCKSDB_LITE - assert(!VerifySstFileChecksum(options, env_options, fname).ok()); + Options options; + options.env = env; + EnvOptions env_options; + Status v = VerifySstFileChecksum(options, env_options, fname); + assert(!v.ok()); #endif + } + return s; +} + +Status TruncateFile(Env* env, const std::string& fname, uint64_t new_length) { + uint64_t old_length; + Status s = env->GetFileSize(fname, &old_length); + if (!s.ok() || new_length == old_length) { + return s; + } + // Do it + std::string contents; + s = ReadFileToString(env, fname, &contents); + if (s.ok()) { + contents.resize(static_cast(new_length), 'b'); + s = WriteStringToFile(env, contents, fname); + } + return s; } } // namespace test diff --git a/test_util/testutil.h b/test_util/testutil.h index 320cccc293..4b1f361e21 100644 --- a/test_util/testutil.h +++ b/test_util/testutil.h @@ -22,9 +22,7 @@ #include "rocksdb/options.h" #include "rocksdb/slice.h" #include "rocksdb/table.h" -#include "table/block_based/block_based_table_factory.h" #include "table/internal_iterator.h" -#include "table/plain/plain_table_factory.h" #include "util/mutexlock.h" namespace ROCKSDB_NAMESPACE { @@ -804,8 +802,9 @@ size_t GetLinesCount(const std::string& fname, const std::string& pattern); // Tries to set TEST_TMPDIR to a directory supporting direct IO. void ResetTmpDirForDirectIO(); - -void CorruptFile(const std::string& fname, int offset, int bytes_to_corrupt); +Status CorruptFile(Env* env, const std::string& fname, int offset, + int bytes_to_corrupt, bool verify_checksum = true); +Status TruncateFile(Env* env, const std::string& fname, uint64_t length); } // namespace test } // namespace ROCKSDB_NAMESPACE diff --git a/util/gflags_compat.h b/util/gflags_compat.h index d5a30ce7e5..ddd3747fa0 100644 --- a/util/gflags_compat.h +++ b/util/gflags_compat.h @@ -3,6 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). +#pragma once #include #ifndef GFLAGS_NAMESPACE diff --git a/utilities/persistent_cache/persistent_cache_test.cc b/utilities/persistent_cache/persistent_cache_test.cc index 5dc090d51d..3ed6e4c02a 100644 --- a/utilities/persistent_cache/persistent_cache_test.cc +++ b/utilities/persistent_cache/persistent_cache_test.cc @@ -253,18 +253,19 @@ TEST_F(PersistentCacheTierTest, DISABLED_TieredCacheInsertWithEviction) { } std::shared_ptr MakeVolatileCache( - const std::string& /*dbname*/) { + Env* /*env*/, const std::string& /*dbname*/) { return std::make_shared(); } -std::shared_ptr MakeBlockCache(const std::string& dbname) { - return NewBlockCache(Env::Default(), dbname); +std::shared_ptr MakeBlockCache(Env* env, + const std::string& dbname) { + return NewBlockCache(env, dbname); } std::shared_ptr MakeTieredCache( - const std::string& dbname) { + Env* env, const std::string& dbname) { const auto memory_size = 1 * 1024 * 1024 * kStressFactor; - return NewTieredCache(Env::Default(), dbname, static_cast(memory_size)); + return NewTieredCache(env, dbname, static_cast(memory_size)); } #ifdef OS_LINUX @@ -442,26 +443,26 @@ void PersistentCacheDBTest::RunTest( // specifically written for Travis. // Now used generally because main tests are too expensive as unit tests. TEST_F(PersistentCacheDBTest, BasicTest) { - RunTest(std::bind(&MakeBlockCache, dbname_), /*max_keys=*/1024, + RunTest(std::bind(&MakeBlockCache, env_, dbname_), /*max_keys=*/1024, /*max_usecase=*/1); } // test table with block page cache // DISABLED for now (very expensive, especially memory) TEST_F(PersistentCacheDBTest, DISABLED_BlockCacheTest) { - RunTest(std::bind(&MakeBlockCache, dbname_)); + RunTest(std::bind(&MakeBlockCache, env_, dbname_)); } // test table with volatile page cache // DISABLED for now (very expensive, especially memory) TEST_F(PersistentCacheDBTest, DISABLED_VolatileCacheTest) { - RunTest(std::bind(&MakeVolatileCache, dbname_)); + RunTest(std::bind(&MakeVolatileCache, env_, dbname_)); } // test table with tiered page cache // DISABLED for now (very expensive, especially memory) TEST_F(PersistentCacheDBTest, DISABLED_TieredCacheTest) { - RunTest(std::bind(&MakeTieredCache, dbname_)); + RunTest(std::bind(&MakeTieredCache, env_, dbname_)); } } // namespace ROCKSDB_NAMESPACE