diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 0ab06489c8..a49b769397 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -79,6 +79,21 @@ class DBOptionsTest : public DBTestBase { #endif // ROCKSDB_LITE }; +TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) { + Options options; + options.track_and_verify_wals_in_manifest = true; + + ImmutableDBOptions db_options(options); + ASSERT_TRUE(db_options.track_and_verify_wals_in_manifest); + + Reopen(options); + ASSERT_TRUE(dbfull()->GetDBOptions().track_and_verify_wals_in_manifest); + + Status s = + dbfull()->SetDBOptions({{"track_and_verify_wals_in_manifest", "false"}}); + ASSERT_FALSE(s.ok()); +} + // RocksDB lite don't support dynamic options. #ifndef ROCKSDB_LITE diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 53c444a307..a85fdf4a6a 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -390,6 +390,20 @@ struct DBOptions { // Default: true bool paranoid_checks = true; + // If true, track WALs in MANIFEST and verify them on recovery. + // + // If a WAL is tracked in MANIFEST but is missing from disk on recovery, + // or the size of the tracked WAL is larger than the WAL's on-disk size, + // an error is reported and recovery is aborted. + // + // If a WAL is not tracked in MANIFEST, then no verification will happen + // during recovery. + // + // Default: false + // FIXME(cheng): This option is part of a work in progress and does not yet + // work + bool track_and_verify_wals_in_manifest = false; + // Use the specified object to interact with the environment, // e.g. to read/write files, schedule background work, etc. In the near // future, support for doing storage operations such as read/write files diff --git a/options/db_options.cc b/options/db_options.cc index 2b97c37db5..c61154a4b9 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -198,6 +198,11 @@ static std::unordered_map {offsetof(struct ImmutableDBOptions, paranoid_checks), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"track_and_verify_wals_in_manifest", + {offsetof(struct ImmutableDBOptions, + track_and_verify_wals_in_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"skip_log_error_on_recovery", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}, @@ -497,6 +502,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) create_missing_column_families(options.create_missing_column_families), error_if_exists(options.error_if_exists), paranoid_checks(options.paranoid_checks), + track_and_verify_wals_in_manifest( + options.track_and_verify_wals_in_manifest), env(options.env), fs(options.env->GetFileSystem()), rate_limiter(options.rate_limiter), @@ -579,6 +586,10 @@ void ImmutableDBOptions::Dump(Logger* log) const { create_if_missing); ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d", paranoid_checks); + ROCKS_LOG_HEADER(log, + " " + "Options.track_and_verify_wals_in_manifest: %d", + track_and_verify_wals_in_manifest); ROCKS_LOG_HEADER(log, " Options.env: %p", env); ROCKS_LOG_HEADER(log, " Options.fs: %s", diff --git a/options/db_options.h b/options/db_options.h index 2684e01b50..631d38ac06 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -23,6 +23,7 @@ struct ImmutableDBOptions { bool create_missing_column_families; bool error_if_exists; bool paranoid_checks; + bool track_and_verify_wals_in_manifest; Env* env; std::shared_ptr fs; std::shared_ptr rate_limiter; diff --git a/options/options_helper.cc b/options/options_helper.cc index 5240c4b945..6bec5e82b1 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -51,6 +51,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, immutable_db_options.create_missing_column_families; options.error_if_exists = immutable_db_options.error_if_exists; options.paranoid_checks = immutable_db_options.paranoid_checks; + options.track_and_verify_wals_in_manifest = + immutable_db_options.track_and_verify_wals_in_manifest; options.env = immutable_db_options.env; options.rate_limiter = immutable_db_options.rate_limiter; options.sst_file_manager = immutable_db_options.sst_file_manager; diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 28e25b5763..3dc87da16e 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -278,6 +278,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "skip_log_error_on_recovery=true;" "writable_file_max_buffer_size=1048576;" "paranoid_checks=true;" + "track_and_verify_wals_in_manifest=true;" "is_fd_close_on_exec=false;" "bytes_per_sync=4295013613;" "strict_bytes_per_sync=true;" diff --git a/options/options_test.cc b/options/options_test.cc index 2ad551eedd..510b9ee1e4 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -109,6 +109,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"create_missing_column_families", "true"}, {"error_if_exists", "false"}, {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, {"max_open_files", "32"}, {"max_total_wal_size", "33"}, {"use_fsync", "true"}, @@ -263,6 +264,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.create_missing_column_families, true); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); ASSERT_EQ(new_db_opt.max_open_files, 32); ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); ASSERT_EQ(new_db_opt.use_fsync, true); @@ -774,6 +776,7 @@ TEST_F(OptionsTest, OldInterfaceTest) { {"create_missing_column_families", "true"}, {"error_if_exists", "false"}, {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, {"max_open_files", "32"}, }; ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); @@ -781,6 +784,7 @@ TEST_F(OptionsTest, OldInterfaceTest) { ASSERT_EQ(new_db_opt.create_missing_column_families, true); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); ASSERT_EQ(new_db_opt.max_open_files, 32); db_options_map["unknown_option"] = "1"; ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); @@ -1620,6 +1624,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { {"create_missing_column_families", "true"}, {"error_if_exists", "false"}, {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, {"max_open_files", "32"}, {"max_total_wal_size", "33"}, {"use_fsync", "true"}, @@ -1768,6 +1773,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.create_missing_column_families, true); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); ASSERT_EQ(new_db_opt.max_open_files, 32); ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); ASSERT_EQ(new_db_opt.use_fsync, true); diff --git a/test_util/testutil.cc b/test_util/testutil.cc index d05bb766e2..24005e782e 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -309,6 +309,7 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { db_opt->error_if_exists = rnd->Uniform(2); db_opt->is_fd_close_on_exec = rnd->Uniform(2); db_opt->paranoid_checks = rnd->Uniform(2); + db_opt->track_and_verify_wals_in_manifest = rnd->Uniform(2); db_opt->skip_log_error_on_recovery = rnd->Uniform(2); db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); db_opt->skip_checking_sst_file_sizes_on_db_open = rnd->Uniform(2);