mirror of https://github.com/facebook/rocksdb.git
bab5f9a6f2
Summary: **Context/Summary:** We are adding new stats to measure behavior of prefetched tail size and look up into this buffer The stat collection is done in FilePrefetchBuffer but only for prefetched tail buffer during table open for now using FilePrefetchBuffer enum. It's cleaner than the alternative of implementing in upper-level call places of FilePrefetchBuffer for table open. It also has the benefit of extensible to other types of FilePrefetchBuffer if needed. See db bench for perf regression concern. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11265 Test Plan: **- Piggyback on existing test** **- rocksdb.table.open.prefetch.tail.miss is harder to UT so I manually set prefetch tail read bytes to be small and run db bench.** ``` ./db_bench -db=/tmp/testdb -statistics=true -benchmarks="fillseq" -key_size=32 -value_size=512 -num=5000 -write_buffer_size=655 -target_file_size_base=655 -disable_auto_compactions=false -compression_type=none -bloom_bits=3 -use_direct_reads=true ``` ``` rocksdb.table.open.prefetch.tail.read.bytes P50 : 4096.000000 P95 : 4096.000000 P99 : 4096.000000 P100 : 4096.000000 COUNT : 225 SUM : 921600 rocksdb.table.open.prefetch.tail.miss COUNT : 91 rocksdb.table.open.prefetch.tail.hit COUNT : 1034 ``` **- No perf regression observed in db_bench** SETUP command: create same db with ~900 files for pre-change/post-change. ``` ./db_bench -db=/tmp/testdb -benchmarks="fillseq" -key_size=32 -value_size=512 -num=500000 -write_buffer_size=655360 -disable_auto_compactions=true -target_file_size_base=16777216 -compression_type=none ``` TEST command 60 runs or til convergence: as suggested by anand1976 and akankshamahajan15, vary `seek_nexts` and `async_io` in testing. ``` ./db_bench -use_existing_db=true -db=/tmp/testdb -statistics=false -cache_size=0 -cache_index_and_filter_blocks=false -benchmarks=seekrandom[-X60] -num=50000 -seek_nexts={10, 500, 1000} -async_io={0|1} -use_direct_reads=true ``` async io = 0, direct io read = true | seek_nexts = 10, 30 runs | seek_nexts = 500, 12 runs | seek_nexts = 1000, 6 runs -- | -- | -- | -- pre-post change | 4776 (± 28) ops/sec; 24.8 (± 0.1) MB/sec | 288 (± 1) ops/sec; 74.8 (± 0.4) MB/sec | 145 (± 4) ops/sec; 75.6 (± 2.2) MB/sec post-change | 4790 (± 32) ops/sec; 24.9 (± 0.2) MB/sec | 288 (± 3) ops/sec; 74.7 (± 0.8) MB/sec | 143 (± 3) ops/sec; 74.5 (± 1.6) MB/sec async io = 1, direct io read = true | seek_nexts = 10, 54 runs | seek_nexts = 500, 6 runs | seek_nexts = 1000, 4 runs -- | -- | -- | -- pre-post change | 3350 (± 36) ops/sec; 17.4 (± 0.2) MB/sec | 264 (± 0) ops/sec; 68.7 (± 0.2) MB/sec | 138 (± 1) ops/sec; 71.8 (± 1.0) MB/sec post-change | 3358 (± 27) ops/sec; 17.4 (± 0.1) MB/sec | 263 (± 2) ops/sec; 68.3 (± 0.8) MB/sec | 139 (± 1) ops/sec; 72.6 (± 0.6) MB/sec Reviewed By: ajkr Differential Revision: D43781467 Pulled By: hx235 fbshipit-source-id: a706a18472a8edb2b952bac3af40eec803537f2a |
||
|---|---|---|
| .. | ||
| backup_engine_options.cc | ||
| backupenginejni.cc | ||
| cache.cc | ||
| cassandra_compactionfilterjni.cc | ||
| cassandra_value_operator.cc | ||
| checkpoint.cc | ||
| clock_cache.cc | ||
| columnfamilyhandle.cc | ||
| compact_range_options.cc | ||
| compaction_filter.cc | ||
| compaction_filter_factory.cc | ||
| compaction_filter_factory_jnicallback.cc | ||
| compaction_filter_factory_jnicallback.h | ||
| compaction_job_info.cc | ||
| compaction_job_stats.cc | ||
| compaction_options.cc | ||
| compaction_options_fifo.cc | ||
| compaction_options_universal.cc | ||
| comparator.cc | ||
| comparatorjnicallback.cc | ||
| comparatorjnicallback.h | ||
| compression_options.cc | ||
| concurrent_task_limiter.cc | ||
| config_options.cc | ||
| cplusplus_to_java_convert.h | ||
| env.cc | ||
| env_options.cc | ||
| event_listener.cc | ||
| event_listener_jnicallback.cc | ||
| event_listener_jnicallback.h | ||
| filter.cc | ||
| ingest_external_file_options.cc | ||
| iterator.cc | ||
| jnicallback.cc | ||
| jnicallback.h | ||
| loggerjnicallback.cc | ||
| loggerjnicallback.h | ||
| lru_cache.cc | ||
| memory_util.cc | ||
| memtablejni.cc | ||
| merge_operator.cc | ||
| native_comparator_wrapper_test.cc | ||
| optimistic_transaction_db.cc | ||
| optimistic_transaction_options.cc | ||
| options.cc | ||
| options_util.cc | ||
| persistent_cache.cc | ||
| portal.h | ||
| ratelimiterjni.cc | ||
| remove_emptyvalue_compactionfilterjni.cc | ||
| restorejni.cc | ||
| rocks_callback_object.cc | ||
| rocksdb_exception_test.cc | ||
| rocksjni.cc | ||
| slice.cc | ||
| snapshot.cc | ||
| sst_file_manager.cc | ||
| sst_file_reader_iterator.cc | ||
| sst_file_readerjni.cc | ||
| sst_file_writerjni.cc | ||
| sst_partitioner.cc | ||
| statistics.cc | ||
| statisticsjni.cc | ||
| statisticsjni.h | ||
| table.cc | ||
| table_filter.cc | ||
| table_filter_jnicallback.cc | ||
| table_filter_jnicallback.h | ||
| testable_event_listener.cc | ||
| thread_status.cc | ||
| trace_writer.cc | ||
| trace_writer_jnicallback.cc | ||
| trace_writer_jnicallback.h | ||
| transaction.cc | ||
| transaction_db.cc | ||
| transaction_db_options.cc | ||
| transaction_log.cc | ||
| transaction_notifier.cc | ||
| transaction_notifier_jnicallback.cc | ||
| transaction_notifier_jnicallback.h | ||
| transaction_options.cc | ||
| ttl.cc | ||
| wal_filter.cc | ||
| wal_filter_jnicallback.cc | ||
| wal_filter_jnicallback.h | ||
| write_batch.cc | ||
| write_batch_test.cc | ||
| write_batch_with_index.cc | ||
| write_buffer_manager.cc | ||
| writebatchhandlerjnicallback.cc | ||
| writebatchhandlerjnicallback.h | ||