diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 1c6be6f9d2..c5294d7c04 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -402,6 +402,9 @@ DECLARE_uint32(lowest_used_cache_tier); DECLARE_bool(enable_custom_split_merge); DECLARE_uint32(adm_policy); DECLARE_bool(enable_memtable_insert_with_hint_prefix_extractor); +DECLARE_bool(check_multiget_consistency); +DECLARE_bool(check_multiget_entity_consistency); +DECLARE_bool(inplace_update_support); constexpr long KB = 1024; constexpr int kRandomValueMaxFactor = 3; diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 53a35666aa..9337f2e160 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -1345,4 +1345,16 @@ DEFINE_bool(enable_memtable_insert_with_hint_prefix_extractor, "If true and FLAGS_prefix_size > 0, set " "Options.memtable_insert_with_hint_prefix_extractor to " "be Options.prefix_extractor"); + +DEFINE_bool(check_multiget_consistency, true, + "If true, check consistency of MultiGet result by comparing it " + "with Get's under a snapshot"); + +DEFINE_bool(check_multiget_entity_consistency, true, + "If true, check consistency of MultiGetEntity result by comparing " + "it GetEntity's under a snapshot"); + +DEFINE_bool(inplace_update_support, + ROCKSDB_NAMESPACE::Options().inplace_update_support, + "Options.inplace_update_support"); #endif // GFLAGS diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index a503da1543..950af11f21 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -3693,6 +3693,7 @@ void InitializeOptionsFromFlags( } options.lowest_used_cache_tier = static_cast(FLAGS_lowest_used_cache_tier); + options.inplace_update_support = FLAGS_inplace_update_support; } void InitializeOptionsGeneral( diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 54147ea9ca..813467efc4 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -249,14 +249,17 @@ int db_stress_tool(int argc, char** argv) { exit(1); } } - if (FLAGS_enable_compaction_filter && + if ((FLAGS_enable_compaction_filter || FLAGS_inplace_update_support) && (FLAGS_acquire_snapshot_one_in > 0 || FLAGS_compact_range_one_in > 0 || FLAGS_iterpercent > 0 || FLAGS_test_batches_snapshots || - FLAGS_test_cf_consistency)) { + FLAGS_test_cf_consistency || FLAGS_check_multiget_consistency || + FLAGS_check_multiget_entity_consistency)) { fprintf( stderr, "Error: acquire_snapshot_one_in, compact_range_one_in, iterpercent, " - "test_batches_snapshots must all be 0 when using compaction filter\n"); + "test_batches_snapshots, test_cf_consistency, " + "check_multiget_consistency, check_multiget_entity_consistency must " + "all be 0 when using compaction filter or inplace update support\n"); exit(1); } if (FLAGS_test_multi_ops_txns) { diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index a94de38971..2fe0df4e78 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -580,13 +580,8 @@ class NonBatchedOpsStressTest : public StressTest { int column_family = rand_column_families[0]; ColumnFamilyHandle* cfh = column_families_[column_family]; int error_count = 0; - // Do a consistency check between Get and MultiGet. Don't do it too - // often as it will slow db_stress down - // - // CompactionFilter can make snapshot non-repeatable by removing keys - // protected by snapshot - bool do_consistency_check = - !FLAGS_enable_compaction_filter && thread->rand.OneIn(4); + + bool do_consistency_check = FLAGS_check_multiget_consistency; ReadOptions readoptionscopy = read_opts; @@ -1075,10 +1070,8 @@ class NonBatchedOpsStressTest : public StressTest { fault_fs_guard->DisableErrorInjection(); } - // CompactionFilter can make snapshot non-repeatable by removing keys - // protected by snapshot - const bool check_get_entity = !FLAGS_enable_compaction_filter && - !error_count && thread->rand.OneIn(4); + const bool check_get_entity = + !error_count && FLAGS_check_multiget_entity_consistency; for (size_t i = 0; i < num_keys; ++i) { const Status& s = statuses[i]; diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index b11532b7d5..483b1ba524 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -74,6 +74,7 @@ default_params = { "destroy_db_initially": 0, "enable_pipelined_write": lambda: random.randint(0, 1), "enable_compaction_filter": lambda: random.choice([0, 0, 0, 1]), + "inplace_update_support": lambda: random.randint(0, 1), "expected_values_dir": lambda: setup_expected_values_dir(), "fail_if_options_file_error": lambda: random.randint(0, 1), "flush_one_in": lambda: random.choice([1000, 1000000]), @@ -280,6 +281,8 @@ default_params = { # TODO(hx235): enable `enable_memtable_insert_with_hint_prefix_extractor` # after fixing the surfaced issue with delete range "enable_memtable_insert_with_hint_prefix_extractor": 0, + "check_multiget_consistency": lambda: random.choice([0, 0, 0, 1]), + "check_multiget_entity_consistency": lambda: random.choice([0, 0, 0, 1]), } _TEST_DIR_ENV_VAR = "TEST_TMPDIR" # If TEST_TMPDIR_EXPECTED is not specified, default value will be TEST_TMPDIR @@ -437,8 +440,9 @@ cf_consistency_params = { "write_buffer_size": 1024 * 1024, "enable_pipelined_write": lambda: random.randint(0, 1), # Snapshots are used heavily in this test mode, while they are incompatible - # with compaction filter. + # with compaction filter, inplace_update_support "enable_compaction_filter": 0, + "inplace_update_support": 0, # `CfConsistencyStressTest::TestIngestExternalFile()` is not implemented. "ingest_external_file_one_in": 0, # `CfConsistencyStressTest::TestIterateAgainstExpected()` is not implemented. @@ -639,6 +643,7 @@ def finalize_and_sanitize(src_params): if dest_params["test_batches_snapshots"] == 1: dest_params["enable_compaction_filter"] = 0 + dest_params["inplace_update_support"] = 0 if dest_params["prefix_size"] < 0: dest_params["prefix_size"] = 1 @@ -699,8 +704,9 @@ def finalize_and_sanitize(src_params): dest_params["enable_pipelined_write"] = 0 if dest_params.get("sst_file_manager_bytes_per_sec", 0) == 0: dest_params["sst_file_manager_bytes_per_truncate"] = 0 - if dest_params.get("enable_compaction_filter", 0) == 1: - # Compaction filter is incompatible with snapshots. Need to avoid taking + if (dest_params.get("enable_compaction_filter", 0) == 1 + or dest_params.get("inplace_update_support", 0) == 1): + # Compaction filter, inplace update support are incompatible with snapshots. Need to avoid taking # snapshots, as well as avoid operations that use snapshots for # verification. dest_params["acquire_snapshot_one_in"] = 0 @@ -708,6 +714,8 @@ def finalize_and_sanitize(src_params): # Give the iterator ops away to reads. dest_params["readpercent"] += dest_params.get("iterpercent", 10) dest_params["iterpercent"] = 0 + dest_params["check_multiget_consistency"] = 0 + dest_params["check_multiget_entity_consistency"] = 0 if dest_params.get("prefix_size") == -1: dest_params["readpercent"] += dest_params.get("prefixpercent", 20) dest_params["prefixpercent"] = 0