dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove DBGet P95/P99 benchmark metrics (#9742) 

Summary:
DBGet p95 and p99 have high variation, remove them for now.
Also increase the iteration to 3 to avoid false positive.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9742

Test Plan: Internal CI

Reviewed By: ajkr

Differential Revision: D35082820

Pulled By: jay-zhuang

fbshipit-source-id: facc1d56b94e54aa8c8852c207aae2ae4e4924b0
This commit is contained in:
Jay Zhuang 2022-03-24 10:08:35 -07:00 committed by Facebook GitHub Bot
parent d583d23d86
commit 18463f8c00
4 changed files with 334 additions and 838 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

834
TARGETS
View File

File diff suppressed because it is too large Load Diff

242
buckifier/bench-slow.json
View File

File diff suppressed because it is too large Load Diff

80
buckifier/bench.json
View File

@ -3,27 +3,23 @@
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 510.387506
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 497.077261
}
@ -135,33 +131,29 @@
"expected_runtime_one_iter": 2438,
"name": "rocksdb_microbench_suite_0",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 510.88842
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 497.546918
}
@ -266,33 +258,29 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_1",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 479.941992
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 513.865636
}
@ -331,7 +319,7 @@
"expected_runtime_one_iter": 2446,
"name": "rocksdb_microbench_suite_2",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -357,27 +345,23 @@
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 471.459789
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 514.201628
}
@ -461,7 +445,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_3",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -487,27 +471,23 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 694.628355
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 517.245591
}
@ -590,7 +570,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_4",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -720,20 +700,18 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_5",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 658.890773
}
@ -822,20 +800,18 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_6",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 505.270116
}
@ -851,14 +827,12 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 532.702176
}
@ -939,7 +913,7 @@
"expected_runtime_one_iter": 2438,
"name": "rocksdb_microbench_suite_7",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -1056,7 +1030,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_8",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -1082,14 +1056,12 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 497.710941
}
@ -1155,46 +1127,40 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_9",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 624.132525
}
],
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 510.156004
}
],
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 551.935026
}
@ -1246,46 +1212,40 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_10",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 507.805668
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 557.749335
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 623.018994
}
@ -1320,7 +1280,7 @@
"expected_runtime_one_iter": 2446,
"name": "rocksdb_microbench_suite_11",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -1336,14 +1296,12 @@
}
],
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 609.749844
}
@ -1359,14 +1317,12 @@
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 557.886298
}
@ -1458,7 +1414,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_12",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -1484,14 +1440,12 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 578.538571
}
@ -1559,7 +1513,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_13",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
},
{
"benchmarks": {
@ -1585,14 +1539,12 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 501.900122
}
@ -1661,6 +1613,6 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_14",
"regression_threshold": 10,
"sl_iterations": 2
"sl_iterations": 3
}
]
]

16
microbench/db_basic_bench.cc
View File

@ -293,9 +293,9 @@ static void DBPut(benchmark::State& state) {
if (enable_statistics) {
HistogramData histogram_data;
options.statistics->histogramData(DB_WRITE, &histogram_data);
state.counters["put_mean"] = histogram_data.average;
state.counters["put_p95"] = histogram_data.percentile95;
state.counters["put_p99"] = histogram_data.percentile99;
state.counters["put_mean"] = histogram_data.average * std::milli::den;
state.counters["put_p95"] = histogram_data.percentile95 * std::milli::den;
state.counters["put_p99"] = histogram_data.percentile99 * std::milli::den;
}
TeardownDB(state, db, options, kg);
@ -445,7 +445,7 @@ static void ManualCompactionArguments(benchmark::internal::Benchmark* b) {
b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics"});
}
BENCHMARK(ManualCompaction)->Iterations(10)->Apply(ManualCompactionArguments);
BENCHMARK(ManualCompaction)->Iterations(1)->Apply(ManualCompactionArguments);
static void ManualFlush(benchmark::State& state) {
uint64_t key_num = state.range(0);
@ -516,7 +516,7 @@ static void ManualFlushArguments(benchmark::internal::Benchmark* b) {
b->ArgNames({"key_num", "per_key_size"});
}
BENCHMARK(ManualFlush)->Iterations(100)->Apply(ManualFlushArguments);
BENCHMARK(ManualFlush)->Iterations(1)->Apply(ManualFlushArguments);
static void DBGet(benchmark::State& state) {
auto compaction_style = static_cast<CompactionStyle>(state.range(0));
@ -599,9 +599,9 @@ static void DBGet(benchmark::State& state) {
if (enable_statistics) {
HistogramData histogram_data;
options.statistics->histogramData(DB_GET, &histogram_data);
state.counters["get_mean"] = histogram_data.average;
state.counters["get_p95"] = histogram_data.percentile95;
state.counters["get_p99"] = histogram_data.percentile99;
state.counters["get_mean"] = histogram_data.average * std::milli::den;
state.counters["get_p95"] = histogram_data.percentile95 * std::milli::den;
state.counters["get_p99"] = histogram_data.percentile99 * std::milli::den;
}
TeardownDB(state, db, options, kg);