dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Clean up compression logging 

Summary: Now we add warnings when user configures compression and the compression is not supported.

Test Plan:
Configured compression to non-supported values. Observed messages in my log:

    2015/03/26-12:17:57.586341 7ffb8a496840 [WARN] Compression type chosen for level 2 is not supported: LZ4. RocksDB will not compress data on level 2.

    2015/03/26-12:19:10.768045 7f36f15c5840 [WARN] Compression type chosen is not supported: LZ4. RocksDB will not compress data.

Reviewers: rven, sdong, yhchiang

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D35979
This commit is contained in:
Igor Canadi 2015-04-06 12:50:44 -07:00
parent e3ee98b38a
commit 5e067a7b19
7 changed files with 125 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
db/column_family.cc
View File

@ -136,7 +136,6 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
result.min_write_buffer_number_to_merge =
std::min(result.min_write_buffer_number_to_merge,
result.max_write_buffer_number - 1);
result.compression_per_level = src.compression_per_level;
if (result.max_mem_compaction_level >= result.num_levels) {
result.max_mem_compaction_level = result.num_levels - 1;
}
@ -155,6 +154,24 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
}
}
if (!src.compression_per_level.empty()) {
for (size_t level = 0; level < src.compression_per_level.size(); ++level) {
if (!CompressionTypeSupported(src.compression_per_level[level])) {
Log(InfoLogLevel::WARN_LEVEL, db_options.info_log,
"Compression type chosen for level %zu is not supported: %s. "
"RocksDB "
"will not compress data on level %zu.",
level, CompressionTypeToString(src.compression_per_level[level]),
level);
}
}
} else if (!CompressionTypeSupported(src.compression)) {
Log(InfoLogLevel::WARN_LEVEL, db_options.info_log,
"Compression type chosen is not supported: %s. RocksDB will not "
"compress data.",
CompressionTypeToString(src.compression));
}
if (result.compaction_style == kCompactionStyleFIFO) {
result.num_levels = 1;
// since we delete level0 files in FIFO compaction when there are too many

29
db/db_impl.cc
View File

@ -67,6 +67,7 @@
#include "util/autovector.h"
#include "util/build_version.h"
#include "util/coding.h"
#include "util/compression.h"
#include "util/db_info_dumper.h"
#include "util/file_util.h"
#include "util/hash_skiplist_rep.h"
@ -189,26 +190,14 @@ CompressionType GetCompressionFlush(const ImmutableCFOptions& ioptions) {
}
void DumpCompressionInfo(Logger* logger) {
Log(InfoLogLevel::INFO_LEVEL, logger, "Snappy "
#ifndef SNAPPY
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "Zlib "
#ifndef ZLIB
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "Bzip "
#ifndef BZIP2
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "LZ4 "
#ifndef LZ4
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "Compression algorithms supported:");
Log(InfoLogLevel::INFO_LEVEL, logger, "\tSnappy supported: %d",
Snappy_Supported());
Log(InfoLogLevel::INFO_LEVEL, logger, "\tZlib supported: %d",
Zlib_Supported());
Log(InfoLogLevel::INFO_LEVEL, logger, "\tBzip supported: %d",
BZip2_Supported());
Log(InfoLogLevel::INFO_LEVEL, logger, "\tLZ4 supported: %d", LZ4_Supported());
}
} // namespace

60
db/db_test.cc
View File

@ -60,41 +60,6 @@
namespace rocksdb {
static bool SnappyCompressionSupported(const CompressionOptions& options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Snappy_Compress(options, in.data(), in.size(), &out);
}
static bool SnappyCompressionSupported() {
CompressionOptions options;
return SnappyCompressionSupported(options);
}
static bool ZlibCompressionSupported(const CompressionOptions& options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Zlib_Compress(options, 2, in.data(), in.size(), &out);
}
static bool BZip2CompressionSupported(const CompressionOptions& options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return BZip2_Compress(options, 2, in.data(), in.size(), &out);
}
static bool LZ4CompressionSupported(const CompressionOptions &options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4_Compress(options, 2, in.data(), in.size(), &out);
}
static bool LZ4HCCompressionSupported(const CompressionOptions &options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4HC_Compress(options, 2, in.data(), in.size(), &out);
}
static std::string RandomString(Random* rnd, int len) {
std::string r;
test::RandomString(rnd, len, &r);
@ -4418,7 +4383,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
}
TEST_F(DBTest, CompressedCache) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
int num_iter = 80;
@ -4542,7 +4507,7 @@ static std::string CompressibleString(Random* rnd, int len) {
}
TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
@ -4611,7 +4576,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
}
TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
Options options;
@ -4655,7 +4620,7 @@ TEST_F(DBTest, FailMoreDbPaths) {
}
TEST_F(DBTest, UniversalCompactionSecondPathRatio) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
Options options;
@ -5348,25 +5313,18 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
options.level0_file_num_compaction_trigger = 3;
options.create_if_missing = true;
if (SnappyCompressionSupported(CompressionOptions(wbits, lev, strategy))) {
if (Snappy_Supported()) {
type = kSnappyCompression;
fprintf(stderr, "using snappy\n");
} else if (ZlibCompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
} else if (Zlib_Supported()) {
type = kZlibCompression;
fprintf(stderr, "using zlib\n");
} else if (BZip2CompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
} else if (BZip2_Supported()) {
type = kBZip2Compression;
fprintf(stderr, "using bzip2\n");
} else if (LZ4CompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
} else if (LZ4_Supported()) {
type = kLZ4Compression;
fprintf(stderr, "using lz4\n");
} else if (LZ4HCCompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
type = kLZ4HCCompression;
fprintf(stderr, "using lz4hc\n");
} else {
fprintf(stderr, "skipping test, compression disabled\n");
return false;
@ -11129,7 +11087,7 @@ TEST_F(DBTest, DynamicLevelMaxBytesBase2) {
}
TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
const int kNKeys = 120;

6
include/rocksdb/options.h
View File

@ -55,6 +55,12 @@ enum CompressionType : char {
kBZip2Compression = 0x3, kLZ4Compression = 0x4, kLZ4HCCompression = 0x5
};
// returns true if RocksDB was correctly linked with compression library and
// supports the compression type
extern bool CompressionTypeSupported(CompressionType compression_type);
// Returns a human-readable name of the compression type
extern const char* CompressionTypeToString(CompressionType compression_type);
enum CompactionStyle : char {
// level based compaction style
kCompactionStyleLevel = 0x0,

80
table/table_test.cc
View File

@ -537,61 +537,6 @@ class DBConstructor: public Constructor {
DB* db_;
};
static bool SnappyCompressionSupported() {
#ifdef SNAPPY
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Snappy_Compress(Options().compression_opts, in.data(), in.size(),
&out);
#else
return false;
#endif
}
static bool ZlibCompressionSupported() {
#ifdef ZLIB
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Zlib_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}
static bool BZip2CompressionSupported() {
#ifdef BZIP2
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return BZip2_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}
static bool LZ4CompressionSupported() {
#ifdef LZ4
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}
static bool LZ4HCCompressionSupported() {
#ifdef LZ4
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4HC_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}
enum TestType {
BLOCK_BASED_TABLE_TEST,
PLAIN_TABLE_SEMI_FIXED_PREFIX,
@ -623,22 +568,20 @@ static std::vector<TestArgs> GenerateArgList() {
// Only add compression if it is supported
std::vector<std::pair<CompressionType, bool>> compression_types;
compression_types.emplace_back(kNoCompression, false);
if (SnappyCompressionSupported()) {
if (Snappy_Supported()) {
compression_types.emplace_back(kSnappyCompression, false);
}
if (ZlibCompressionSupported()) {
if (Zlib_Supported()) {
compression_types.emplace_back(kZlibCompression, false);
compression_types.emplace_back(kZlibCompression, true);
}
if (BZip2CompressionSupported()) {
if (BZip2_Supported()) {
compression_types.emplace_back(kBZip2Compression, false);
compression_types.emplace_back(kBZip2Compression, true);
}
if (LZ4CompressionSupported()) {
if (LZ4_Supported()) {
compression_types.emplace_back(kLZ4Compression, false);
compression_types.emplace_back(kLZ4Compression, true);
}
if (LZ4HCCompressionSupported()) {
compression_types.emplace_back(kLZ4HCCompression, false);
compression_types.emplace_back(kLZ4HCCompression, true);
}
@ -1918,13 +1861,13 @@ static void DoCompressionTest(CompressionType comp) {
TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) {
std::vector<CompressionType> compression_state;
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
fprintf(stderr, "skipping snappy compression tests\n");
} else {
compression_state.push_back(kSnappyCompression);
}
if (!ZlibCompressionSupported()) {
if (!Zlib_Supported()) {
fprintf(stderr, "skipping zlib compression tests\n");
} else {
compression_state.push_back(kZlibCompression);
@ -1932,22 +1875,17 @@ TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) {
// TODO(kailiu) DoCompressionTest() doesn't work with BZip2.
/*
if (!BZip2CompressionSupported()) {
if (!BZip2_Supported()) {
fprintf(stderr, "skipping bzip2 compression tests\n");
} else {
compression_state.push_back(kBZip2Compression);
}
*/
if (!LZ4CompressionSupported()) {
fprintf(stderr, "skipping lz4 compression tests\n");
if (!LZ4_Supported()) {
fprintf(stderr, "skipping lz4 and lz4hc compression tests\n");
} else {
compression_state.push_back(kLZ4Compression);
}
if (!LZ4HCCompressionSupported()) {
fprintf(stderr, "skipping lz4hc compression tests\n");
} else {
compression_state.push_back(kLZ4HCCompression);
}

28
util/compression.h
View File

@ -34,6 +34,34 @@
namespace rocksdb {
inline bool Snappy_Supported() {
#ifdef SNAPPY
return true;
#endif
return false;
}
inline bool Zlib_Supported() {
#ifdef ZLIB
return true;
#endif
return false;
}
inline bool BZip2_Supported() {
#ifdef BZIP2
return true;
#endif
return false;
}
inline bool LZ4_Supported() {
#ifdef LZ4
return true;
#endif
return false;
}
// compress_format_version can have two values:
// 1 -- decompressed sizes for BZip2 and Zlib are not included in the compressed
// block. Also, decompressed sizes for LZ4 are encoded in platform-dependent

50
util/options.cc
View File

@ -29,6 +29,7 @@
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
#include "table/block_based_table_factory.h"
#include "util/compression.h"
#include "util/statistics.h"
#include "util/xfunc.h"
@ -383,11 +384,12 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
Log(log, " Options.max_write_buffer_number: %d", max_write_buffer_number);
if (!compression_per_level.empty()) {
for (unsigned int i = 0; i < compression_per_level.size(); i++) {
Log(log," Options.compression[%d]: %d",
i, compression_per_level[i]);
}
Log(log, " Options.compression[%d]: %s", i,
CompressionTypeToString(compression_per_level[i]));
}
} else {
Log(log," Options.compression: %d", compression);
Log(log, " Options.compression: %s",
CompressionTypeToString(compression));
}
Log(log," Options.prefix_extractor: %s",
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
@ -543,6 +545,46 @@ Options::PrepareForBulkLoad()
return this;
}
const char* CompressionTypeToString(CompressionType compression_type) {
switch (compression_type) {
case kNoCompression:
return "NoCompression";
case kSnappyCompression:
return "Snappy";
case kZlibCompression:
return "Zlib";
case kBZip2Compression:
return "BZip2";
case kLZ4Compression:
return "LZ4";
case kLZ4HCCompression:
return "LZ4HC";
default:
assert(false);
return "";
}
}
bool CompressionTypeSupported(CompressionType compression_type) {
switch (compression_type) {
case kNoCompression:
return true;
case kSnappyCompression:
return Snappy_Supported();
case kZlibCompression:
return Zlib_Supported();
case kBZip2Compression:
return BZip2_Supported();
case kLZ4Compression:
return LZ4_Supported();
case kLZ4HCCompression:
return LZ4_Supported();
default:
assert(false);
return false;
}
}
#ifndef ROCKSDB_LITE
// Optimization functions
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(