diff --git a/db/db_statistics_test.cc b/db/db_statistics_test.cc index ed80760518..adc2b36bb2 100644 --- a/db/db_statistics_test.cc +++ b/db/db_statistics_test.cc @@ -20,131 +20,115 @@ class DBStatisticsTest : public DBTestBase { }; TEST_F(DBStatisticsTest, CompressionStatsTest) { - CompressionType type; + for (CompressionType type : GetSupportedCompressions()) { + if (type == kNoCompression) { + continue; + } + if (type == kBZip2Compression) { + // Weird behavior in this test + continue; + } + SCOPED_TRACE("Compression type: " + std::to_string(type)); - if (Snappy_Supported()) { - type = kSnappyCompression; - fprintf(stderr, "using snappy\n"); - } else if (Zlib_Supported()) { - type = kZlibCompression; - fprintf(stderr, "using zlib\n"); - } else if (BZip2_Supported()) { - type = kBZip2Compression; - fprintf(stderr, "using bzip2\n"); - } else if (LZ4_Supported()) { - type = kLZ4Compression; - fprintf(stderr, "using lz4\n"); - } else if (XPRESS_Supported()) { - type = kXpressCompression; - fprintf(stderr, "using xpress\n"); - } else if (ZSTD_Supported()) { - type = kZSTD; - fprintf(stderr, "using ZSTD\n"); - } else { - fprintf(stderr, "skipping test, compression disabled\n"); - return; + Options options = CurrentOptions(); + options.compression = type; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex); + BlockBasedTableOptions bbto; + bbto.enable_index_compression = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + auto PopStat = [&](Tickers t) -> uint64_t { + return options.statistics->getAndResetTickerCount(t); + }; + + int kNumKeysWritten = 100; + double compress_to = 0.5; + // About three KVs per block + int len = static_cast(BlockBasedTableOptions().block_size / 3); + int uncomp_est = kNumKeysWritten * (len + 20); + + Random rnd(301); + std::string buf; + + // Check that compressions occur and are counted when compression is turned + // on + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK( + Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); + } + ASSERT_OK(Flush()); + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10); + EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); + + // And decompressions + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + } + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10); + EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); + + // Check when compression is rejected. + DestroyAndReopen(options); + + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len))); + } + ASSERT_OK(Flush()); + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + } + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); + + // Check when compression is disabled. + options.compression = kNoCompression; + DestroyAndReopen(options); + + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len))); + } + ASSERT_OK(Flush()); + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + } + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); } - - Options options = CurrentOptions(); - options.compression = type; - options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); - options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex); - BlockBasedTableOptions bbto; - bbto.enable_index_compression = false; - options.table_factory.reset(NewBlockBasedTableFactory(bbto)); - DestroyAndReopen(options); - - auto PopStat = [&](Tickers t) -> uint64_t { - return options.statistics->getAndResetTickerCount(t); - }; - - int kNumKeysWritten = 100; - double compress_to = 0.5; - // About three KVs per block - int len = static_cast(BlockBasedTableOptions().block_size / 3); - int uncomp_est = kNumKeysWritten * (len + 20); - - Random rnd(301); - std::string buf; - - // Check that compressions occur and are counted when compression is turned on - for (int i = 0; i < kNumKeysWritten; ++i) { - ASSERT_OK( - Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); - } - ASSERT_OK(Flush()); - EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED)); - EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10); - EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO), - uncomp_est / 10); - - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); - EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); - EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); - - // And decompressions - for (int i = 0; i < kNumKeysWritten; ++i) { - auto r = Get(Key(i)); - } - EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED)); - EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10); - EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM), - uncomp_est / 10); - - EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED)); - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); - - // Check when compression is rejected. - compress_to = 0.95; - DestroyAndReopen(options); - - for (int i = 0; i < kNumKeysWritten; ++i) { - ASSERT_OK( - Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); - } - ASSERT_OK(Flush()); - for (int i = 0; i < kNumKeysWritten; ++i) { - auto r = Get(Key(i)); - } - EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); - EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED), - uncomp_est / 10); - - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED)); - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED)); - EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); - EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); - - // Check when compression is disabled. - options.compression = kNoCompression; - DestroyAndReopen(options); - - for (int i = 0; i < kNumKeysWritten; ++i) { - ASSERT_OK( - Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); - } - ASSERT_OK(Flush()); - for (int i = 0; i < kNumKeysWritten; ++i) { - auto r = Get(Key(i)); - } - EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); - EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED), - uncomp_est / 10); - - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED)); - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); - EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO)); - EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED)); - EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); - EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); } TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) { diff --git a/table/table_test.cc b/table/table_test.cc index 8f0f4b1f1a..5b1eced54c 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -5081,63 +5081,49 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) { } TEST_P(BlockBasedTableTest, CompressionRatioThreshold) { - Options options; - if (Snappy_Supported()) { - options.compression = kSnappyCompression; - fprintf(stderr, "using snappy\n"); - } else if (Zlib_Supported()) { - options.compression = kZlibCompression; - fprintf(stderr, "using zlib\n"); - } else if (BZip2_Supported()) { - options.compression = kBZip2Compression; - fprintf(stderr, "using bzip2\n"); - } else if (LZ4_Supported()) { - options.compression = kLZ4Compression; - fprintf(stderr, "using lz4\n"); - } else if (XPRESS_Supported()) { - options.compression = kXpressCompression; - fprintf(stderr, "using xpress\n"); - } else if (ZSTD_Supported()) { - options.compression = kZSTD; - fprintf(stderr, "using ZSTD\n"); - } else { - fprintf(stderr, "skipping test, compression disabled\n"); - return; - } + for (CompressionType type : GetSupportedCompressions()) { + if (type == kNoCompression) { + continue; + } + SCOPED_TRACE("Compression type: " + std::to_string(type)); - BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); - int len = 10000; - Random rnd(301); - std::vector keys; - stl_wrappers::KVMap kvmap; + Options options; + options.compression = type; - // Test the max_compressed_bytes_per_kb option - for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) { - SCOPED_TRACE("threshold=" + std::to_string(threshold)); - options.compression_opts.max_compressed_bytes_per_kb = threshold; - ImmutableOptions ioptions(options); - MutableCFOptions moptions(options); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + int len = 10000; + Random rnd(301); + std::vector keys; + stl_wrappers::KVMap kvmap; - for (double compressible_to : {0.25, 0.75}) { - SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to)); - TableConstructor c(BytewiseComparator(), - true /* convert_to_internal_key_ */); - std::string buf; - c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf)); + // Test the max_compressed_bytes_per_kb option + for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) { + SCOPED_TRACE("threshold=" + std::to_string(threshold)); + options.compression_opts.max_compressed_bytes_per_kb = threshold; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); - // write an SST file - c.Finish(options, ioptions, moptions, table_options, - GetPlainInternalComparator(options.comparator), &keys, &kvmap); + for (double compressible_to : {0.25, 0.75}) { + SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to)); + TableConstructor c(BytewiseComparator(), + true /* convert_to_internal_key_ */); + std::string buf; + c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf)); - size_t table_file_size = c.TEST_GetSink()->contents().size(); - size_t approx_sst_overhead = 1000; - if (compressible_to < threshold / 1024.0) { - // Should be compressed - EXPECT_NEAR2(len * compressible_to + approx_sst_overhead, - table_file_size, len / 10); - } else { - // Should not be compressed - EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10); + // write an SST file + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + size_t table_file_size = c.TEST_GetSink()->contents().size(); + size_t approx_sst_overhead = 1000; + if (compressible_to < threshold / 1024.0) { + // Should be compressed (substantial variance depending on algorithm) + EXPECT_NEAR2(len * compressible_to + approx_sst_overhead, + table_file_size, len / 8); + } else { + // Should not be compressed + EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10); + } } } } diff --git a/test_util/testutil.cc b/test_util/testutil.cc index b3dfc0830e..2500f926ee 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -76,7 +76,7 @@ extern Slice CompressibleString(Random* rnd, double compressed_fraction, int len, std::string* dst) { int raw = static_cast(len * compressed_fraction); if (raw < 1) raw = 1; - std::string raw_data = rnd->RandomString(raw); + std::string raw_data = rnd->RandomBinaryString(raw); // Duplicate the random data until we have filled "len" bytes dst->clear();