mirror of https://github.com/facebook/rocksdb.git
Fix compression tests when snappy not available (#11396)
Summary: Tweak some bounds and things, and reduce risk of surprise results by running on all supported compressions (mostly). Also improves the precise compressibility of CompressibleString by using RandomBinaryString. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11396 Test Plan: updated tests Reviewed By: ltamasi Differential Revision: D45211938 Pulled By: pdillinger fbshipit-source-id: 9dc1dd8574a60a9364efe18558be66d31a35598b
This commit is contained in:
Peter Dillinger
Facebook GitHub Bot
parent
d79be3dca2
commit
fb63d9b4ee
|
|
@ -20,131 +20,115 @@ class DBStatisticsTest : public DBTestBase {
|
|||
};
|
||||
|
||||
TEST_F(DBStatisticsTest, CompressionStatsTest) {
|
||||
CompressionType type;
|
||||
for (CompressionType type : GetSupportedCompressions()) {
|
||||
if (type == kNoCompression) {
|
||||
continue;
|
||||
}
|
||||
if (type == kBZip2Compression) {
|
||||
// Weird behavior in this test
|
||||
continue;
|
||||
}
|
||||
SCOPED_TRACE("Compression type: " + std::to_string(type));
|
||||
|
||||
if (Snappy_Supported()) {
|
||||
type = kSnappyCompression;
|
||||
fprintf(stderr, "using snappy\n");
|
||||
} else if (Zlib_Supported()) {
|
||||
type = kZlibCompression;
|
||||
fprintf(stderr, "using zlib\n");
|
||||
} else if (BZip2_Supported()) {
|
||||
type = kBZip2Compression;
|
||||
fprintf(stderr, "using bzip2\n");
|
||||
} else if (LZ4_Supported()) {
|
||||
type = kLZ4Compression;
|
||||
fprintf(stderr, "using lz4\n");
|
||||
} else if (XPRESS_Supported()) {
|
||||
type = kXpressCompression;
|
||||
fprintf(stderr, "using xpress\n");
|
||||
} else if (ZSTD_Supported()) {
|
||||
type = kZSTD;
|
||||
fprintf(stderr, "using ZSTD\n");
|
||||
} else {
|
||||
fprintf(stderr, "skipping test, compression disabled\n");
|
||||
return;
|
||||
Options options = CurrentOptions();
|
||||
options.compression = type;
|
||||
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||
options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
|
||||
BlockBasedTableOptions bbto;
|
||||
bbto.enable_index_compression = false;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
auto PopStat = [&](Tickers t) -> uint64_t {
|
||||
return options.statistics->getAndResetTickerCount(t);
|
||||
};
|
||||
|
||||
int kNumKeysWritten = 100;
|
||||
double compress_to = 0.5;
|
||||
// About three KVs per block
|
||||
int len = static_cast<int>(BlockBasedTableOptions().block_size / 3);
|
||||
int uncomp_est = kNumKeysWritten * (len + 20);
|
||||
|
||||
Random rnd(301);
|
||||
std::string buf;
|
||||
|
||||
// Check that compressions occur and are counted when compression is turned
|
||||
// on
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
ASSERT_OK(
|
||||
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
|
||||
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
|
||||
|
||||
// And decompressions
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
auto r = Get(Key(i));
|
||||
}
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
|
||||
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
|
||||
|
||||
// Check when compression is rejected.
|
||||
DestroyAndReopen(options);
|
||||
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
auto r = Get(Key(i));
|
||||
}
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
|
||||
|
||||
// Check when compression is disabled.
|
||||
options.compression = kNoCompression;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
auto r = Get(Key(i));
|
||||
}
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
|
||||
}
|
||||
|
||||
Options options = CurrentOptions();
|
||||
options.compression = type;
|
||||
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||
options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
|
||||
BlockBasedTableOptions bbto;
|
||||
bbto.enable_index_compression = false;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
auto PopStat = [&](Tickers t) -> uint64_t {
|
||||
return options.statistics->getAndResetTickerCount(t);
|
||||
};
|
||||
|
||||
int kNumKeysWritten = 100;
|
||||
double compress_to = 0.5;
|
||||
// About three KVs per block
|
||||
int len = static_cast<int>(BlockBasedTableOptions().block_size / 3);
|
||||
int uncomp_est = kNumKeysWritten * (len + 20);
|
||||
|
||||
Random rnd(301);
|
||||
std::string buf;
|
||||
|
||||
// Check that compressions occur and are counted when compression is turned on
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
ASSERT_OK(
|
||||
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
|
||||
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
|
||||
|
||||
// And decompressions
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
auto r = Get(Key(i));
|
||||
}
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
|
||||
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
|
||||
|
||||
// Check when compression is rejected.
|
||||
compress_to = 0.95;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
ASSERT_OK(
|
||||
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
auto r = Get(Key(i));
|
||||
}
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
|
||||
|
||||
// Check when compression is disabled.
|
||||
options.compression = kNoCompression;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
ASSERT_OK(
|
||||
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||
auto r = Get(Key(i));
|
||||
}
|
||||
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
|
||||
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
|
||||
uncomp_est / 10);
|
||||
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
|
||||
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
|
||||
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
|
||||
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
|
||||
}
|
||||
|
||||
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {
|
||||
|
|
|
|||
|
|
@ -5081,63 +5081,49 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
|
|||
}
|
||||
|
||||
TEST_P(BlockBasedTableTest, CompressionRatioThreshold) {
|
||||
Options options;
|
||||
if (Snappy_Supported()) {
|
||||
options.compression = kSnappyCompression;
|
||||
fprintf(stderr, "using snappy\n");
|
||||
} else if (Zlib_Supported()) {
|
||||
options.compression = kZlibCompression;
|
||||
fprintf(stderr, "using zlib\n");
|
||||
} else if (BZip2_Supported()) {
|
||||
options.compression = kBZip2Compression;
|
||||
fprintf(stderr, "using bzip2\n");
|
||||
} else if (LZ4_Supported()) {
|
||||
options.compression = kLZ4Compression;
|
||||
fprintf(stderr, "using lz4\n");
|
||||
} else if (XPRESS_Supported()) {
|
||||
options.compression = kXpressCompression;
|
||||
fprintf(stderr, "using xpress\n");
|
||||
} else if (ZSTD_Supported()) {
|
||||
options.compression = kZSTD;
|
||||
fprintf(stderr, "using ZSTD\n");
|
||||
} else {
|
||||
fprintf(stderr, "skipping test, compression disabled\n");
|
||||
return;
|
||||
}
|
||||
for (CompressionType type : GetSupportedCompressions()) {
|
||||
if (type == kNoCompression) {
|
||||
continue;
|
||||
}
|
||||
SCOPED_TRACE("Compression type: " + std::to_string(type));
|
||||
|
||||
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
|
||||
int len = 10000;
|
||||
Random rnd(301);
|
||||
std::vector<std::string> keys;
|
||||
stl_wrappers::KVMap kvmap;
|
||||
Options options;
|
||||
options.compression = type;
|
||||
|
||||
// Test the max_compressed_bytes_per_kb option
|
||||
for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) {
|
||||
SCOPED_TRACE("threshold=" + std::to_string(threshold));
|
||||
options.compression_opts.max_compressed_bytes_per_kb = threshold;
|
||||
ImmutableOptions ioptions(options);
|
||||
MutableCFOptions moptions(options);
|
||||
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
|
||||
int len = 10000;
|
||||
Random rnd(301);
|
||||
std::vector<std::string> keys;
|
||||
stl_wrappers::KVMap kvmap;
|
||||
|
||||
for (double compressible_to : {0.25, 0.75}) {
|
||||
SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to));
|
||||
TableConstructor c(BytewiseComparator(),
|
||||
true /* convert_to_internal_key_ */);
|
||||
std::string buf;
|
||||
c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf));
|
||||
// Test the max_compressed_bytes_per_kb option
|
||||
for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) {
|
||||
SCOPED_TRACE("threshold=" + std::to_string(threshold));
|
||||
options.compression_opts.max_compressed_bytes_per_kb = threshold;
|
||||
ImmutableOptions ioptions(options);
|
||||
MutableCFOptions moptions(options);
|
||||
|
||||
// write an SST file
|
||||
c.Finish(options, ioptions, moptions, table_options,
|
||||
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
|
||||
for (double compressible_to : {0.25, 0.75}) {
|
||||
SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to));
|
||||
TableConstructor c(BytewiseComparator(),
|
||||
true /* convert_to_internal_key_ */);
|
||||
std::string buf;
|
||||
c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf));
|
||||
|
||||
size_t table_file_size = c.TEST_GetSink()->contents().size();
|
||||
size_t approx_sst_overhead = 1000;
|
||||
if (compressible_to < threshold / 1024.0) {
|
||||
// Should be compressed
|
||||
EXPECT_NEAR2(len * compressible_to + approx_sst_overhead,
|
||||
table_file_size, len / 10);
|
||||
} else {
|
||||
// Should not be compressed
|
||||
EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10);
|
||||
// write an SST file
|
||||
c.Finish(options, ioptions, moptions, table_options,
|
||||
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
|
||||
|
||||
size_t table_file_size = c.TEST_GetSink()->contents().size();
|
||||
size_t approx_sst_overhead = 1000;
|
||||
if (compressible_to < threshold / 1024.0) {
|
||||
// Should be compressed (substantial variance depending on algorithm)
|
||||
EXPECT_NEAR2(len * compressible_to + approx_sst_overhead,
|
||||
table_file_size, len / 8);
|
||||
} else {
|
||||
// Should not be compressed
|
||||
EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ extern Slice CompressibleString(Random* rnd, double compressed_fraction,
|
|||
int len, std::string* dst) {
|
||||
int raw = static_cast<int>(len * compressed_fraction);
|
||||
if (raw < 1) raw = 1;
|
||||
std::string raw_data = rnd->RandomString(raw);
|
||||
std::string raw_data = rnd->RandomBinaryString(raw);
|
||||
|
||||
// Duplicate the random data until we have filled "len" bytes
|
||||
dst->clear();
|
||||
|
|
|
|||
Loading…
Reference in New Issue