From 4fa2c017190a99a9ad44562b12aeef6f6d2a1d16 Mon Sep 17 00:00:00 2001 From: Jay Huh Date: Mon, 21 Aug 2023 12:14:57 -0700 Subject: [PATCH] Replace existing waitforcompaction with new WaitForCompact API in db_bench_tool (#11727) Summary: As the new API to wait for compaction is available (https://github.com/facebook/rocksdb/issues/11436), we can now replace the existing logic of waiting in db_bench_tool with the new API. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11727 Test Plan: ``` ./db_bench --benchmarks="fillrandom,compactall,waitforcompaction,readrandom" ``` **Before change** ``` Set seed to 1692635571470041 because --seed was 0 Initializing RocksDB Options from the specified file Initializing RocksDB Options from command-line flags Integrated BlobDB: blob cache disabled RocksDB: version 8.6.0 Date: Mon Aug 21 09:33:40 2023 CPU: 80 * Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz CPUCache: 28160 KB Keys: 16 bytes each (+ 0 bytes user-defined timestamp) Values: 100 bytes each (50 bytes after compression) Entries: 1000000 Prefix: 0 bytes Keys per prefix: 0 RawSize: 110.6 MB (estimated) FileSize: 62.9 MB (estimated) Write rate: 0 bytes/second Read rate: 0 ops/second Compression: Snappy Compression sampling rate: 0 Memtablerep: SkipListFactory Perf Level: 1 WARNING: Optimization is disabled: benchmarks unnecessarily slow WARNING: Assertions are enabled; benchmarks unnecessarily slow ------------------------------------------------ Initializing RocksDB Options from the specified file Initializing RocksDB Options from command-line flags Integrated BlobDB: blob cache disabled DB path: [/tmp/rocksdbtest-226125/dbbench] fillrandom : 51.826 micros/op 19295 ops/sec 51.826 seconds 1000000 operations; 2.1 MB/s waitforcompaction(/tmp/rocksdbtest-226125/dbbench): started waitforcompaction(/tmp/rocksdbtest-226125/dbbench): finished waitforcompaction(/tmp/rocksdbtest-226125/dbbench): started waitforcompaction(/tmp/rocksdbtest-226125/dbbench): finished DB path: [/tmp/rocksdbtest-226125/dbbench] readrandom : 39.042 micros/op 25613 ops/sec 39.042 seconds 1000000 operations; 1.8 MB/s (632886 of 1000000 found) ``` **After change** ``` Set seed to 1692636574431745 because --seed was 0 Initializing RocksDB Options from the specified file Initializing RocksDB Options from command-line flags Integrated BlobDB: blob cache disabled RocksDB: version 8.6.0 Date: Mon Aug 21 09:49:34 2023 CPU: 80 * Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz CPUCache: 28160 KB Keys: 16 bytes each (+ 0 bytes user-defined timestamp) Values: 100 bytes each (50 bytes after compression) Entries: 1000000 Prefix: 0 bytes Keys per prefix: 0 RawSize: 110.6 MB (estimated) FileSize: 62.9 MB (estimated) Write rate: 0 bytes/second Read rate: 0 ops/second Compression: Snappy Compression sampling rate: 0 Memtablerep: SkipListFactory Perf Level: 1 WARNING: Optimization is disabled: benchmarks unnecessarily slow WARNING: Assertions are enabled; benchmarks unnecessarily slow ------------------------------------------------ Initializing RocksDB Options from the specified file Initializing RocksDB Options from command-line flags Integrated BlobDB: blob cache disabled DB path: [/tmp/rocksdbtest-226125/dbbench] fillrandom : 51.271 micros/op 19504 ops/sec 51.271 seconds 1000000 operations; 2.2 MB/s waitforcompaction(/tmp/rocksdbtest-226125/dbbench): started waitforcompaction(/tmp/rocksdbtest-226125/dbbench): finished with status (OK) DB path: [/tmp/rocksdbtest-226125/dbbench] readrandom : 39.264 micros/op 25468 ops/sec 39.264 seconds 1000000 operations; 1.8 MB/s (632921 of 1000000 found) ``` Reviewed By: ajkr Differential Revision: D48524667 Pulled By: jaykorean fbshipit-source-id: 1052a15b2ed79a35165ec4d9998d0454b2552ef4 --- tools/db_bench_tool.cc | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index db66d0721e..0a133289b3 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -8114,58 +8114,24 @@ class Benchmark { } void WaitForCompactionHelper(DBWithColumnFamilies& db) { - // This is an imperfect way of waiting for compaction. The loop and sleep - // is done because a thread that finishes a compaction job should get a - // chance to pickup a new compaction job. - - std::vector keys = {DB::Properties::kMemTableFlushPending, - DB::Properties::kNumRunningFlushes, - DB::Properties::kCompactionPending, - DB::Properties::kNumRunningCompactions}; - fprintf(stdout, "waitforcompaction(%s): started\n", db.db->GetName().c_str()); - while (true) { - bool retry = false; + Status s = db.db->WaitForCompact(WaitForCompactOptions()); - for (const auto& k : keys) { - uint64_t v; - if (!db.db->GetIntProperty(k, &v)) { - fprintf(stderr, "waitforcompaction(%s): GetIntProperty(%s) failed\n", - db.db->GetName().c_str(), k.c_str()); - exit(1); - } else if (v > 0) { - fprintf(stdout, - "waitforcompaction(%s): active(%s). Sleep 10 seconds\n", - db.db->GetName().c_str(), k.c_str()); - FLAGS_env->SleepForMicroseconds(10 * 1000000); - retry = true; - break; - } - } - - if (!retry) { - fprintf(stdout, "waitforcompaction(%s): finished\n", - db.db->GetName().c_str()); - return; - } - } + fprintf(stdout, "waitforcompaction(%s): finished with status (%s)\n", + db.db->GetName().c_str(), s.ToString().c_str()); } void WaitForCompaction() { // Give background threads a chance to wake FLAGS_env->SleepForMicroseconds(5 * 1000000); - // I am skeptical that this check race free. I hope that checking twice - // reduces the chance. if (db_.db != nullptr) { WaitForCompactionHelper(db_); - WaitForCompactionHelper(db_); } else { for (auto& db_with_cfh : multi_dbs_) { WaitForCompactionHelper(db_with_cfh); - WaitForCompactionHelper(db_with_cfh); } } }