Speed up BlockTest.BlockReadAmpBitmap

Summary:
BlockTest.BlockReadAmpBitmap is too slow and times out in some environments. Speed it up by:
(1) improve the way the verification is done. With this it is 5 times faster
(2) run fewer tests for large blocks. This cut it down by another 10 times.
Now it can finish in similar time as other tests.
Closes https://github.com/facebook/rocksdb/pull/3313

Differential Revision: D6643711

Pulled By: siying

fbshipit-source-id: c2397d666eab5421a78ca87e1e45491e0f832a6d
This commit is contained in:
Siying Dong 2018-01-02 10:34:51 -08:00 committed by Facebook Github Bot
parent b5c99cc908
commit ccc095a016
2 changed files with 44 additions and 22 deletions

View File

@ -961,9 +961,7 @@ Status WinWritableFile::Sync() {
return SyncImpl(); return SyncImpl();
} }
Status WinWritableFile::Fsync() { Status WinWritableFile::Fsync() { return SyncImpl(); }
return SyncImpl();
}
bool WinWritableFile::IsSyncThreadSafe() const { return true; } bool WinWritableFile::IsSyncThreadSafe() const { return true; }

View File

@ -229,18 +229,47 @@ class BlockReadAmpBitmapSlowAndAccurate {
marked_ranges_.emplace(end_offset, start_offset); marked_ranges_.emplace(end_offset, start_offset);
} }
void ResetCheckSequence() { iter_valid_ = false; }
// Return true if any byte in this range was Marked // Return true if any byte in this range was Marked
// This does linear search from the previous position. When calling
// multiple times, `offset` needs to be incremental to get correct results.
// Call ResetCheckSequence() to reset it.
bool IsPinMarked(size_t offset) { bool IsPinMarked(size_t offset) {
auto it = marked_ranges_.lower_bound( if (iter_valid_) {
// Has existing iterator, try linear search from
// the iterator.
for (int i = 0; i < 64; i++) {
if (offset < iter_->second) {
return false;
}
if (offset <= iter_->first) {
return true;
}
iter_++;
if (iter_ == marked_ranges_.end()) {
iter_valid_ = false;
return false;
}
}
}
// Initial call or have linear searched too many times.
// Do binary search.
iter_ = marked_ranges_.lower_bound(
std::make_pair(offset, static_cast<size_t>(0))); std::make_pair(offset, static_cast<size_t>(0)));
if (it == marked_ranges_.end()) { if (iter_ == marked_ranges_.end()) {
iter_valid_ = false;
return false; return false;
} }
return offset <= it->first && offset >= it->second; iter_valid_ = true;
return offset <= iter_->first && offset >= iter_->second;
} }
private: private:
std::set<std::pair<size_t, size_t>> marked_ranges_; std::set<std::pair<size_t, size_t>> marked_ranges_;
std::set<std::pair<size_t, size_t>>::iterator iter_;
bool iter_valid_ = false;
}; };
TEST_F(BlockTest, BlockReadAmpBitmap) { TEST_F(BlockTest, BlockReadAmpBitmap) {
@ -251,18 +280,16 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
}); });
SyncPoint::GetInstance()->EnableProcessing(); SyncPoint::GetInstance()->EnableProcessing();
std::vector<size_t> block_sizes = { std::vector<size_t> block_sizes = {
1, // 1 byte 1, // 1 byte
32, // 32 bytes 32, // 32 bytes
61, // 61 bytes 61, // 61 bytes
64, // 64 bytes 64, // 64 bytes
512, // 0.5 KB 512, // 0.5 KB
1024, // 1 KB 1024, // 1 KB
1024 * 4, // 4 KB 1024 * 4, // 4 KB
1024 * 10, // 10 KB 1024 * 10, // 10 KB
1024 * 50, // 50 KB 1024 * 50, // 50 KB
1024 * 1024, // 1 MB 1024 * 1024 * 4, // 5 MB
1024 * 1024 * 4, // 4 MB
1024 * 1024 * 50, // 10 MB
777, 777,
124653, 124653,
}; };
@ -278,10 +305,6 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
if (block_size % kBytesPerBit != 0) { if (block_size % kBytesPerBit != 0) {
needed_bits++; needed_bits++;
} }
size_t bitmap_size = needed_bits / 32;
if (needed_bits % 32 != 0) {
bitmap_size++;
}
ASSERT_EQ(stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES), block_size); ASSERT_EQ(stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES), block_size);
@ -309,6 +332,7 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
} }
for (size_t i = 0; i < random_entries.size(); i++) { for (size_t i = 0; i < random_entries.size(); i++) {
read_amp_slow_and_accurate.ResetCheckSequence();
auto &current_entry = random_entries[rnd.Next() % random_entries.size()]; auto &current_entry = random_entries[rnd.Next() % random_entries.size()];
read_amp_bitmap.Mark(static_cast<uint32_t>(current_entry.first), read_amp_bitmap.Mark(static_cast<uint32_t>(current_entry.first),