mirror of https://github.com/facebook/rocksdb.git
Speed up BlockTest.BlockReadAmpBitmap
Summary: BlockTest.BlockReadAmpBitmap is too slow and times out in some environments. Speed it up by: (1) improve the way the verification is done. With this it is 5 times faster (2) run fewer tests for large blocks. This cut it down by another 10 times. Now it can finish in similar time as other tests. Closes https://github.com/facebook/rocksdb/pull/3313 Differential Revision: D6643711 Pulled By: siying fbshipit-source-id: c2397d666eab5421a78ca87e1e45491e0f832a6d
This commit is contained in:
parent
b5c99cc908
commit
ccc095a016
|
@ -961,9 +961,7 @@ Status WinWritableFile::Sync() {
|
|||
return SyncImpl();
|
||||
}
|
||||
|
||||
Status WinWritableFile::Fsync() {
|
||||
return SyncImpl();
|
||||
}
|
||||
Status WinWritableFile::Fsync() { return SyncImpl(); }
|
||||
|
||||
bool WinWritableFile::IsSyncThreadSafe() const { return true; }
|
||||
|
||||
|
|
|
@ -229,18 +229,47 @@ class BlockReadAmpBitmapSlowAndAccurate {
|
|||
marked_ranges_.emplace(end_offset, start_offset);
|
||||
}
|
||||
|
||||
void ResetCheckSequence() { iter_valid_ = false; }
|
||||
|
||||
// Return true if any byte in this range was Marked
|
||||
// This does linear search from the previous position. When calling
|
||||
// multiple times, `offset` needs to be incremental to get correct results.
|
||||
// Call ResetCheckSequence() to reset it.
|
||||
bool IsPinMarked(size_t offset) {
|
||||
auto it = marked_ranges_.lower_bound(
|
||||
if (iter_valid_) {
|
||||
// Has existing iterator, try linear search from
|
||||
// the iterator.
|
||||
for (int i = 0; i < 64; i++) {
|
||||
if (offset < iter_->second) {
|
||||
return false;
|
||||
}
|
||||
if (offset <= iter_->first) {
|
||||
return true;
|
||||
}
|
||||
|
||||
iter_++;
|
||||
if (iter_ == marked_ranges_.end()) {
|
||||
iter_valid_ = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Initial call or have linear searched too many times.
|
||||
// Do binary search.
|
||||
iter_ = marked_ranges_.lower_bound(
|
||||
std::make_pair(offset, static_cast<size_t>(0)));
|
||||
if (it == marked_ranges_.end()) {
|
||||
if (iter_ == marked_ranges_.end()) {
|
||||
iter_valid_ = false;
|
||||
return false;
|
||||
}
|
||||
return offset <= it->first && offset >= it->second;
|
||||
iter_valid_ = true;
|
||||
return offset <= iter_->first && offset >= iter_->second;
|
||||
}
|
||||
|
||||
private:
|
||||
std::set<std::pair<size_t, size_t>> marked_ranges_;
|
||||
std::set<std::pair<size_t, size_t>>::iterator iter_;
|
||||
bool iter_valid_ = false;
|
||||
};
|
||||
|
||||
TEST_F(BlockTest, BlockReadAmpBitmap) {
|
||||
|
@ -251,18 +280,16 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
|
|||
});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
std::vector<size_t> block_sizes = {
|
||||
1, // 1 byte
|
||||
32, // 32 bytes
|
||||
61, // 61 bytes
|
||||
64, // 64 bytes
|
||||
512, // 0.5 KB
|
||||
1024, // 1 KB
|
||||
1024 * 4, // 4 KB
|
||||
1024 * 10, // 10 KB
|
||||
1024 * 50, // 50 KB
|
||||
1024 * 1024, // 1 MB
|
||||
1024 * 1024 * 4, // 4 MB
|
||||
1024 * 1024 * 50, // 10 MB
|
||||
1, // 1 byte
|
||||
32, // 32 bytes
|
||||
61, // 61 bytes
|
||||
64, // 64 bytes
|
||||
512, // 0.5 KB
|
||||
1024, // 1 KB
|
||||
1024 * 4, // 4 KB
|
||||
1024 * 10, // 10 KB
|
||||
1024 * 50, // 50 KB
|
||||
1024 * 1024 * 4, // 5 MB
|
||||
777,
|
||||
124653,
|
||||
};
|
||||
|
@ -278,10 +305,6 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
|
|||
if (block_size % kBytesPerBit != 0) {
|
||||
needed_bits++;
|
||||
}
|
||||
size_t bitmap_size = needed_bits / 32;
|
||||
if (needed_bits % 32 != 0) {
|
||||
bitmap_size++;
|
||||
}
|
||||
|
||||
ASSERT_EQ(stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES), block_size);
|
||||
|
||||
|
@ -309,6 +332,7 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
|
|||
}
|
||||
|
||||
for (size_t i = 0; i < random_entries.size(); i++) {
|
||||
read_amp_slow_and_accurate.ResetCheckSequence();
|
||||
auto ¤t_entry = random_entries[rnd.Next() % random_entries.size()];
|
||||
|
||||
read_amp_bitmap.Mark(static_cast<uint32_t>(current_entry.first),
|
||||
|
|
Loading…
Reference in New Issue