dolysis
/
snappy
mirror of https://github.com/google/snappy.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add LZ4 as a benchmark option. Snappy is starting to look really good compared to LZ4. LZ4 is considered the fastest solution by many on internet. We now see that Snappy is actually becoming very competitive with compression a little faster and decompression slower but certainly not terribly slower. 

PiperOrigin-RevId: 343140860
This commit is contained in:
Snappy Team 2020-11-18 21:10:24 +00:00 committed by Victor Costan
parent e4a6e97b91
commit 616b8229b6
3 changed files with 45 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -108,6 +108,7 @@ check_include_file("windows.h" HAVE_WINDOWS_H)
include(CheckLibraryExists)
check_library_exists(z zlibVersion "" HAVE_LIBZ)
check_library_exists(lzo2 lzo1x_1_15_compress "" HAVE_LIBLZO2)
check_library_exists(lz4 LZ4_compress_default "" HAVE_LIBLZ4)
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("/arch:AVX" HAVE_VISUAL_STUDIO_ARCH_AVX)
@ -268,6 +269,9 @@ if(SNAPPY_BUILD_TESTS)
if(HAVE_LIBLZO2)
target_link_libraries(snappy_unittest lzo2)
endif(HAVE_LIBLZO2)
if(HAVE_LIBLZ4)
target_link_libraries(snappy_unittest lz4)
endif(HAVE_LIBLZ4)
target_include_directories(snappy_unittest
BEFORE PRIVATE

4
snappy-test.h
View File

@ -107,6 +107,10 @@
#include "lzo/lzo1x.h"
#endif
#ifdef HAVE_LIBLZ4
#include "lz4.h"
#endif
namespace {
namespace file {

46
snappy_unittest.cc
View File

@ -51,6 +51,7 @@ DEFINE_bool(zlib, false,
"Run zlib compression (http://www.zlib.net)");
DEFINE_bool(lzo, false,
"Run LZO compression (http://www.oberhumer.com/opensource/lzo/)");
DEFINE_bool(lz4, false, "Run LZ4 compression (https://github.com/lz4/lz4)");
DEFINE_bool(snappy, true, "Run snappy compression");
DEFINE_bool(write_compressed, false,
@ -116,13 +117,9 @@ using DataEndingAtUnreadablePage = std::string;
#endif
enum CompressorType {
ZLIB, LZO, SNAPPY
};
enum CompressorType { ZLIB, LZO, LZ4, SNAPPY };
const char* names[] = {
"ZLIB", "LZO", "SNAPPY"
};
const char* names[] = {"ZLIB", "LZO", "LZ4", "SNAPPY"};
static size_t MinimumRequiredOutputSpace(size_t input_size,
CompressorType comp) {
@ -137,6 +134,11 @@ static size_t MinimumRequiredOutputSpace(size_t input_size,
return input_size + input_size/64 + 16 + 3;
#endif // LZO_VERSION
#ifdef LZ4_VERSION_NUMBER
case LZ4:
return LZ4_compressBound(input_size);
#endif // LZ4_VERSION_NUMBER
case SNAPPY:
return snappy::MaxCompressedLength(input_size);
@ -196,6 +198,19 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
}
#endif // LZO_VERSION
#ifdef LZ4_VERSION_NUMBER
case LZ4: {
lzo_uint destlen = compressed->size();
destlen = LZ4_compress_default(input, string_as_array(compressed),
input_size, destlen);
CHECK(destlen != 0);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // LZ4_VERSION_NUMBER
case SNAPPY: {
size_t destlen;
snappy::RawCompress(input, input_size,
@ -250,6 +265,18 @@ static bool Uncompress(const std::string& compressed, CompressorType comp,
}
#endif // LZO_VERSION
#ifdef LZ4_VERSION_NUMBER
case LZ4: {
output->resize(size);
ZLib zlib;
uLongf destlen = output->size();
destlen = LZ4_decompress_safe(compressed.data(), string_as_array(output),
compressed.size(), destlen);
CHECK(destlen != 0);
CHECK_EQ(size, destlen);
break;
}
#endif // LZ4_VERSION_NUMBER
case SNAPPY: {
snappy::RawUncompress(compressed.data(), compressed.size(),
string_as_array(output));
@ -1223,9 +1250,10 @@ static void MeasureFile(const char* fname) {
for (int len = start_len; len <= end_len; ++len) {
const char* const input = fullinput.data();
int repeats = (FLAGS_bytes + len) / (len + 1);
if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10);
if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10);
if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10);
if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024 << 10);
if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024 << 10);
if (FLAGS_lz4) Measure(input, len, LZ4, repeats, 1024 << 10);
if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096 << 10);
// For block-size based measurements
if (0 && FLAGS_snappy) {