dolysis
/
snappy
mirror of https://github.com/google/snappy.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move the logic from ComputeTable into the unit test, which means it's run 

automatically together with the other tests, and also removes the stray
function ComputeTable() (which was never referenced by anything else
in the open-source version, causing compiler warnings for some)
out of the core library.

Fixes public issue 96.

A=sesse
R=sanjay
This commit is contained in:
Steinar H. Gunderson 2015-08-19 11:37:51 +02:00
parent d80342922c
commit 0852af7606
3 changed files with 167 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
snappy-internal.h
View File

@ -144,6 +144,68 @@ static inline int FindMatchLength(const char* s1,
} }
#endif #endif
// Lookup tables for decompression code. Give --snappy_dump_decompression_table
// to the unit test to recompute char_table.
enum {
LITERAL = 0,
COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode
COPY_2_BYTE_OFFSET = 2,
COPY_4_BYTE_OFFSET = 3
};
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
// Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits
static const uint32 wordmask[] = {
0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu
};
// Data stored per entry in lookup table:
// Range Bits-used Description
// ------------------------------------
// 1..64 0..7 Literal/copy length encoded in opcode byte
// 0..7 8..10 Copy offset encoded in opcode byte / 256
// 0..4 11..13 Extra bytes after opcode
//
// We use eight bits for the length even though 7 would have sufficed
// because of efficiency reasons:
// (1) Extracting a byte is faster than a bit-field
// (2) It properly aligns copy offset so we do not need a <<8
static const uint16 char_table[256] = {
0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002,
0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004,
0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006,
0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008,
0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a,
0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c,
0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e,
0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010,
0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012,
0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014,
0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016,
0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018,
0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a,
0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c,
0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e,
0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020,
0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022,
0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024,
0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026,
0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028,
0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a,
0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c,
0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e,
0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030,
0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032,
0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034,
0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036,
0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038,
0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a,
0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c,
0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e,
0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040
};
} // end namespace internal } // end namespace internal
} // end namespace snappy } // end namespace snappy

172
snappy.cc
View File

@ -39,6 +39,14 @@
namespace snappy { namespace snappy {
using internal::COPY_1_BYTE_OFFSET;
using internal::COPY_2_BYTE_OFFSET;
using internal::COPY_4_BYTE_OFFSET;
using internal::LITERAL;
using internal::char_table;
using internal::kMaximumTagLength;
using internal::wordmask;
// Any hash function will produce a valid compressed bitstream, but a good // Any hash function will produce a valid compressed bitstream, but a good
// hash function reduces the number of collisions and thus yields better // hash function reduces the number of collisions and thus yields better
// compression for compressible input, and more speed for incompressible // compression for compressible input, and more speed for incompressible
@ -76,14 +84,6 @@ size_t MaxCompressedLength(size_t source_len) {
return 32 + source_len + source_len/6; return 32 + source_len + source_len/6;
} }
enum {
LITERAL = 0,
COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode
COPY_2_BYTE_OFFSET = 2,
COPY_4_BYTE_OFFSET = 3
};
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
// Copy "len" bytes from "src" to "op", one byte at a time. Used for // Copy "len" bytes from "src" to "op", one byte at a time. Used for
// handling COPY operations where the input and output regions may // handling COPY operations where the input and output regions may
// overlap. For example, suppose: // overlap. For example, suppose:
@ -493,162 +493,6 @@ char* CompressFragment(const char* input,
// bool TryFastAppend(const char* ip, size_t available, size_t length); // bool TryFastAppend(const char* ip, size_t available, size_t length);
// }; // };
// -----------------------------------------------------------------------
// Lookup table for decompression code. Generated by ComputeTable() below.
// -----------------------------------------------------------------------
// Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits
static const uint32 wordmask[] = {
0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu
};
// Data stored per entry in lookup table:
// Range Bits-used Description
// ------------------------------------
// 1..64 0..7 Literal/copy length encoded in opcode byte
// 0..7 8..10 Copy offset encoded in opcode byte / 256
// 0..4 11..13 Extra bytes after opcode
//
// We use eight bits for the length even though 7 would have sufficed
// because of efficiency reasons:
// (1) Extracting a byte is faster than a bit-field
// (2) It properly aligns copy offset so we do not need a <<8
static const uint16 char_table[256] = {
0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002,
0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004,
0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006,
0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008,
0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a,
0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c,
0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e,
0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010,
0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012,
0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014,
0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016,
0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018,
0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a,
0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c,
0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e,
0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020,
0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022,
0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024,
0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026,
0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028,
0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a,
0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c,
0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e,
0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030,
0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032,
0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034,
0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036,
0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038,
0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a,
0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c,
0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e,
0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040
};
// In debug mode, allow optional computation of the table at startup.
// Also, check that the decompression table is correct.
#ifndef NDEBUG
DEFINE_bool(snappy_dump_decompression_table, false,
"If true, we print the decompression table at startup.");
static uint16 MakeEntry(unsigned int extra,
unsigned int len,
unsigned int copy_offset) {
// Check that all of the fields fit within the allocated space
assert(extra == (extra & 0x7)); // At most 3 bits
assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits
assert(len == (len & 0x7f)); // At most 7 bits
return len | (copy_offset << 8) | (extra << 11);
}
static void ComputeTable() {
uint16 dst[256];
// Place invalid entries in all places to detect missing initialization
int assigned = 0;
for (int i = 0; i < 256; i++) {
dst[i] = 0xffff;
}
// Small LITERAL entries. We store (len-1) in the top 6 bits.
for (unsigned int len = 1; len <= 60; len++) {
dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0);
assigned++;
}
// Large LITERAL entries. We use 60..63 in the high 6 bits to
// encode the number of bytes of length info that follow the opcode.
for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) {
// We set the length field in the lookup table to 1 because extra
// bytes encode len-1.
dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0);
assigned++;
}
// COPY_1_BYTE_OFFSET.
//
// The tag byte in the compressed data stores len-4 in 3 bits, and
// offset/256 in 5 bits. offset%256 is stored in the next byte.
//
// This format is used for length in range [4..11] and offset in
// range [0..2047]
for (unsigned int len = 4; len < 12; len++) {
for (unsigned int offset = 0; offset < 2048; offset += 256) {
dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] =
MakeEntry(1, len, offset>>8);
assigned++;
}
}
// COPY_2_BYTE_OFFSET.
// Tag contains len-1 in top 6 bits, and offset in next two bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0);
assigned++;
}
// COPY_4_BYTE_OFFSET.
// Tag contents len-1 in top 6 bits, and offset in next four bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0);
assigned++;
}
// Check that each entry was initialized exactly once.
if (assigned != 256) {
fprintf(stderr, "ComputeTable: assigned only %d of 256\n", assigned);
abort();
}
for (int i = 0; i < 256; i++) {
if (dst[i] == 0xffff) {
fprintf(stderr, "ComputeTable: did not assign byte %d\n", i);
abort();
}
}
if (FLAGS_snappy_dump_decompression_table) {
printf("static const uint16 char_table[256] = {\n ");
for (int i = 0; i < 256; i++) {
printf("0x%04x%s",
dst[i],
((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", ")));
}
printf("};\n");
}
// Check that computed table matched recorded table
for (int i = 0; i < 256; i++) {
if (dst[i] != char_table[i]) {
fprintf(stderr, "ComputeTable: byte %d: computed (%x), expect (%x)\n",
i, static_cast<int>(dst[i]), static_cast<int>(char_table[i]));
abort();
}
}
}
#endif /* !NDEBUG */
// Helper class for decompression // Helper class for decompression
class SnappyDecompressor { class SnappyDecompressor {

97
snappy_unittest.cc
View File

@ -64,6 +64,9 @@ DEFINE_bool(write_compressed, false,
DEFINE_bool(write_uncompressed, false, DEFINE_bool(write_uncompressed, false,
"Write uncompressed versions of each file to <file>.uncomp"); "Write uncompressed versions of each file to <file>.uncomp");
DEFINE_bool(snappy_dump_decompression_table, false,
"If true, we print the decompression table during tests.");
namespace snappy { namespace snappy {
@ -1160,6 +1163,100 @@ TEST(Snappy, FindMatchLengthRandom) {
} }
} }
static uint16 MakeEntry(unsigned int extra,
unsigned int len,
unsigned int copy_offset) {
// Check that all of the fields fit within the allocated space
assert(extra == (extra & 0x7)); // At most 3 bits
assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits
assert(len == (len & 0x7f)); // At most 7 bits
return len | (copy_offset << 8) | (extra << 11);
}
// Check that the decompression table is correct, and optionally print out
// the computed one.
TEST(Snappy, VerifyCharTable) {
using snappy::internal::LITERAL;
using snappy::internal::COPY_1_BYTE_OFFSET;
using snappy::internal::COPY_2_BYTE_OFFSET;
using snappy::internal::COPY_4_BYTE_OFFSET;
using snappy::internal::char_table;
using snappy::internal::wordmask;
uint16 dst[256];
// Place invalid entries in all places to detect missing initialization
int assigned = 0;
for (int i = 0; i < 256; i++) {
dst[i] = 0xffff;
}
// Small LITERAL entries. We store (len-1) in the top 6 bits.
for (unsigned int len = 1; len <= 60; len++) {
dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0);
assigned++;
}
// Large LITERAL entries. We use 60..63 in the high 6 bits to
// encode the number of bytes of length info that follow the opcode.
for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) {
// We set the length field in the lookup table to 1 because extra
// bytes encode len-1.
dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0);
assigned++;
}
// COPY_1_BYTE_OFFSET.
//
// The tag byte in the compressed data stores len-4 in 3 bits, and
// offset/256 in 5 bits. offset%256 is stored in the next byte.
//
// This format is used for length in range [4..11] and offset in
// range [0..2047]
for (unsigned int len = 4; len < 12; len++) {
for (unsigned int offset = 0; offset < 2048; offset += 256) {
dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] =
MakeEntry(1, len, offset>>8);
assigned++;
}
}
// COPY_2_BYTE_OFFSET.
// Tag contains len-1 in top 6 bits, and offset in next two bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0);
assigned++;
}
// COPY_4_BYTE_OFFSET.
// Tag contents len-1 in top 6 bits, and offset in next four bytes.
for (unsigned int len = 1; len <= 64; len++) {
dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0);
assigned++;
}
// Check that each entry was initialized exactly once.
EXPECT_EQ(256, assigned) << "Assigned only " << assigned << " of 256";
for (int i = 0; i < 256; i++) {
EXPECT_NE(0xffff, dst[i]) << "Did not assign byte " << i;
}
if (FLAGS_snappy_dump_decompression_table) {
printf("static const uint16 char_table[256] = {\n ");
for (int i = 0; i < 256; i++) {
printf("0x%04x%s",
dst[i],
((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", ")));
}
printf("};\n");
}
// Check that computed table matched recorded table.
for (int i = 0; i < 256; i++) {
EXPECT_EQ(dst[i], char_table[i]) << "Mismatch in byte " << i;
}
}
static void CompressFile(const char* fname) { static void CompressFile(const char* fname) {
string fullinput; string fullinput;
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));