snappy/snappy-sinksource.h

183 lines
7.1 KiB
C
Raw Normal View History

// Copyright 2011 Google Inc. All Rights Reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#define THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_
#include <stddef.h>
namespace snappy {
// A Sink is an interface that consumes a sequence of bytes.
class Sink {
public:
Sink() { }
virtual ~Sink();
// Append "bytes[0,n-1]" to this.
virtual void Append(const char* bytes, size_t n) = 0;
// Returns a writable buffer of the specified length for appending.
// May return a pointer to the caller-owned scratch buffer which
// must have at least the indicated length. The returned buffer is
// only valid until the next operation on this Sink.
//
// After writing at most "length" bytes, call Append() with the
// pointer returned from this function and the number of bytes
// written. Many Append() implementations will avoid copying
// bytes if this function returned an internal buffer.
//
// If a non-scratch buffer is returned, the caller may only pass a
// prefix of it to Append(). That is, it is not correct to pass an
// interior pointer of the returned array to Append().
//
// The default implementation always returns the scratch buffer.
virtual char* GetAppendBuffer(size_t length, char* scratch);
Add support for Uncompress(source, sink). Various changes to allow Uncompress(source, sink) to get the same performance as the different variants of Uncompress to Cord/DataBuffer/String/FlatBuffer. Changes to efficiently support Uncompress(source, sink) -------- a) For strings - we add support to StringByteSink to do GetAppendBuffer so we can write to it without copying. b) For flat array buffers, we do GetAppendBuffer and see if we can get a full buffer. With the above changes we get performance with ByteSource/ByteSink that is very close to directly using flat arrays and strings. We add various benchmark cases to demonstrate that. Orthogonal change ------------------ Add support for TryFastAppend() for SnappyScatteredWriter. Benchmark results are below CPU: Intel Core2 dL1:32KB dL2:4096KB Benchmark Time(ns) CPU(ns) Iterations ----------------------------------------------------- BM_UFlat/0 109065 108996 6410 896.0MB/s html BM_UFlat/1 1012175 1012343 691 661.4MB/s urls BM_UFlat/2 26775 26771 26149 4.4GB/s jpg BM_UFlat/3 48947 48940 14363 1.8GB/s pdf BM_UFlat/4 441029 440835 1589 886.1MB/s html4 BM_UFlat/5 39861 39880 17823 588.3MB/s cp BM_UFlat/6 18315 18300 38126 581.1MB/s c BM_UFlat/7 5254 5254 100000 675.4MB/s lsp BM_UFlat/8 1568060 1567376 447 626.6MB/s xls BM_UFlat/9 337512 337734 2073 429.5MB/s txt1 BM_UFlat/10 287269 287054 2434 415.9MB/s txt2 BM_UFlat/11 890098 890219 787 457.2MB/s txt3 BM_UFlat/12 1186593 1186863 590 387.2MB/s txt4 BM_UFlat/13 573927 573318 1000 853.7MB/s bin BM_UFlat/14 64250 64294 10000 567.2MB/s sum BM_UFlat/15 7301 7300 96153 552.2MB/s man BM_UFlat/16 109617 109636 6375 1031.5MB/s pb BM_UFlat/17 364438 364497 1921 482.3MB/s gaviota BM_UFlatSink/0 108518 108465 6450 900.4MB/s html BM_UFlatSink/1 991952 991997 705 675.0MB/s urls BM_UFlatSink/2 26815 26798 26065 4.4GB/s jpg BM_UFlatSink/3 49127 49122 14255 1.8GB/s pdf BM_UFlatSink/4 436674 436731 1604 894.4MB/s html4 BM_UFlatSink/5 39738 39733 17345 590.5MB/s cp BM_UFlatSink/6 18413 18416 37962 577.4MB/s c BM_UFlatSink/7 5677 5676 100000 625.2MB/s lsp BM_UFlatSink/8 1552175 1551026 451 633.2MB/s xls BM_UFlatSink/9 338526 338489 2065 428.5MB/s txt1 BM_UFlatSink/10 289387 289307 2420 412.6MB/s txt2 BM_UFlatSink/11 893803 893706 783 455.4MB/s txt3 BM_UFlatSink/12 1195919 1195459 586 384.4MB/s txt4 BM_UFlatSink/13 559637 559779 1000 874.3MB/s bin BM_UFlatSink/14 65073 65094 10000 560.2MB/s sum BM_UFlatSink/15 7618 7614 92823 529.5MB/s man BM_UFlatSink/16 110085 110121 6352 1027.0MB/s pb BM_UFlatSink/17 369196 368915 1896 476.5MB/s gaviota BM_UValidate/0 46954 46957 14899 2.0GB/s html BM_UValidate/1 500621 500868 1000 1.3GB/s urls BM_UValidate/2 283 283 2481447 417.2GB/s jpg BM_UValidate/3 16230 16228 43137 5.4GB/s pdf BM_UValidate/4 189129 189193 3701 2.0GB/s html4 A=uday R=sanjay
2015-06-22 14:03:28 +00:00
// For higher performance, Sink implementations can provide custom
// AppendAndTakeOwnership() and GetAppendBufferVariable() methods.
// These methods can reduce the number of copies done during
// compression/decompression.
// Append "bytes[0,n-1] to the sink. Takes ownership of "bytes"
// and calls the deleter function as (*deleter)(deleter_arg, bytes, n)
// to free the buffer. deleter function must be non NULL.
//
// The default implementation just calls Append and frees "bytes".
// Other implementations may avoid a copy while appending the buffer.
virtual void AppendAndTakeOwnership(
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
void *deleter_arg);
// Returns a writable buffer for appending and writes the buffer's capacity to
// *allocated_size. Guarantees *allocated_size >= min_size.
// May return a pointer to the caller-owned scratch buffer which must have
// scratch_size >= min_size.
//
// The returned buffer is only valid until the next operation
// on this ByteSink.
//
// After writing at most *allocated_size bytes, call Append() with the
// pointer returned from this function and the number of bytes written.
// Many Append() implementations will avoid copying bytes if this function
// returned an internal buffer.
//
// If the sink implementation allocates or reallocates an internal buffer,
// it should use the desired_size_hint if appropriate. If a caller cannot
// provide a reasonable guess at the desired capacity, it should set
// desired_size_hint = 0.
//
// If a non-scratch buffer is returned, the caller may only pass
// a prefix to it to Append(). That is, it is not correct to pass an
// interior pointer to Append().
//
// The default implementation always returns the scratch buffer.
virtual char* GetAppendBufferVariable(
size_t min_size, size_t desired_size_hint, char* scratch,
size_t scratch_size, size_t* allocated_size);
private:
// No copying
Sink(const Sink&);
void operator=(const Sink&);
};
// A Source is an interface that yields a sequence of bytes
class Source {
public:
Source() { }
virtual ~Source();
// Return the number of bytes left to read from the source
virtual size_t Available() const = 0;
// Peek at the next flat region of the source. Does not reposition
// the source. The returned region is empty iff Available()==0.
//
// Returns a pointer to the beginning of the region and store its
// length in *len.
//
// The returned region is valid until the next call to Skip() or
// until this object is destroyed, whichever occurs first.
//
// The returned region may be larger than Available() (for example
// if this ByteSource is a view on a substring of a larger source).
// The caller is responsible for ensuring that it only reads the
// Available() bytes.
virtual const char* Peek(size_t* len) = 0;
// Skip the next n bytes. Invalidates any buffer returned by
// a previous call to Peek().
// REQUIRES: Available() >= n
virtual void Skip(size_t n) = 0;
private:
// No copying
Source(const Source&);
void operator=(const Source&);
};
// A Source implementation that yields the contents of a flat array
class ByteArraySource : public Source {
public:
ByteArraySource(const char* p, size_t n) : ptr_(p), left_(n) { }
~ByteArraySource() override;
size_t Available() const override;
const char* Peek(size_t* len) override;
void Skip(size_t n) override;
private:
const char* ptr_;
size_t left_;
};
// A Sink implementation that writes to a flat array without any bound checks.
class UncheckedByteArraySink : public Sink {
public:
explicit UncheckedByteArraySink(char* dest) : dest_(dest) { }
~UncheckedByteArraySink() override;
void Append(const char* data, size_t n) override;
char* GetAppendBuffer(size_t len, char* scratch) override;
char* GetAppendBufferVariable(
Add support for Uncompress(source, sink). Various changes to allow Uncompress(source, sink) to get the same performance as the different variants of Uncompress to Cord/DataBuffer/String/FlatBuffer. Changes to efficiently support Uncompress(source, sink) -------- a) For strings - we add support to StringByteSink to do GetAppendBuffer so we can write to it without copying. b) For flat array buffers, we do GetAppendBuffer and see if we can get a full buffer. With the above changes we get performance with ByteSource/ByteSink that is very close to directly using flat arrays and strings. We add various benchmark cases to demonstrate that. Orthogonal change ------------------ Add support for TryFastAppend() for SnappyScatteredWriter. Benchmark results are below CPU: Intel Core2 dL1:32KB dL2:4096KB Benchmark Time(ns) CPU(ns) Iterations ----------------------------------------------------- BM_UFlat/0 109065 108996 6410 896.0MB/s html BM_UFlat/1 1012175 1012343 691 661.4MB/s urls BM_UFlat/2 26775 26771 26149 4.4GB/s jpg BM_UFlat/3 48947 48940 14363 1.8GB/s pdf BM_UFlat/4 441029 440835 1589 886.1MB/s html4 BM_UFlat/5 39861 39880 17823 588.3MB/s cp BM_UFlat/6 18315 18300 38126 581.1MB/s c BM_UFlat/7 5254 5254 100000 675.4MB/s lsp BM_UFlat/8 1568060 1567376 447 626.6MB/s xls BM_UFlat/9 337512 337734 2073 429.5MB/s txt1 BM_UFlat/10 287269 287054 2434 415.9MB/s txt2 BM_UFlat/11 890098 890219 787 457.2MB/s txt3 BM_UFlat/12 1186593 1186863 590 387.2MB/s txt4 BM_UFlat/13 573927 573318 1000 853.7MB/s bin BM_UFlat/14 64250 64294 10000 567.2MB/s sum BM_UFlat/15 7301 7300 96153 552.2MB/s man BM_UFlat/16 109617 109636 6375 1031.5MB/s pb BM_UFlat/17 364438 364497 1921 482.3MB/s gaviota BM_UFlatSink/0 108518 108465 6450 900.4MB/s html BM_UFlatSink/1 991952 991997 705 675.0MB/s urls BM_UFlatSink/2 26815 26798 26065 4.4GB/s jpg BM_UFlatSink/3 49127 49122 14255 1.8GB/s pdf BM_UFlatSink/4 436674 436731 1604 894.4MB/s html4 BM_UFlatSink/5 39738 39733 17345 590.5MB/s cp BM_UFlatSink/6 18413 18416 37962 577.4MB/s c BM_UFlatSink/7 5677 5676 100000 625.2MB/s lsp BM_UFlatSink/8 1552175 1551026 451 633.2MB/s xls BM_UFlatSink/9 338526 338489 2065 428.5MB/s txt1 BM_UFlatSink/10 289387 289307 2420 412.6MB/s txt2 BM_UFlatSink/11 893803 893706 783 455.4MB/s txt3 BM_UFlatSink/12 1195919 1195459 586 384.4MB/s txt4 BM_UFlatSink/13 559637 559779 1000 874.3MB/s bin BM_UFlatSink/14 65073 65094 10000 560.2MB/s sum BM_UFlatSink/15 7618 7614 92823 529.5MB/s man BM_UFlatSink/16 110085 110121 6352 1027.0MB/s pb BM_UFlatSink/17 369196 368915 1896 476.5MB/s gaviota BM_UValidate/0 46954 46957 14899 2.0GB/s html BM_UValidate/1 500621 500868 1000 1.3GB/s urls BM_UValidate/2 283 283 2481447 417.2GB/s jpg BM_UValidate/3 16230 16228 43137 5.4GB/s pdf BM_UValidate/4 189129 189193 3701 2.0GB/s html4 A=uday R=sanjay
2015-06-22 14:03:28 +00:00
size_t min_size, size_t desired_size_hint, char* scratch,
fix compile warnings due to missing override specifiers When building Snappy with compiler option `-Wsuggest-override` set via the CMAKE_CXX_FLAGS, compilation produces warnings in `snappy-sinksource.h`: ``` In file included from ./snappy-fork/snappy-sinksource.cc:32: ./snappy-fork/snappy-sinksource.h:150:18: error: ‘virtual size_t snappy::ByteArraySource::Available() const’ can be marked override [-Werror=suggest-override] 150 | virtual size_t Available() const; | ^~~~~~~~~ ./snappy-fork/snappy-sinksource.h:151:23: error: ‘virtual const char* snappy::ByteArraySource::Peek(size_t*)’ can be marked override [-Werror=suggest-override] 151 | virtual const char* Peek(size_t* len); | ^~~~ ./snappy-fork/snappy-sinksource.h:152:16: error: ‘virtual void snappy::ByteArraySource::Skip(size_t)’ can be marked override [-Werror=suggest-override] 152 | virtual void Skip(size_t n); | ^~~~ ./snappy-fork/snappy-sinksource.h:163:16: error: ‘virtual void snappy::UncheckedByteArraySink::Append(const char*, size_t)’ can be marked override [-Werror=suggest-override] 163 | virtual void Append(const char* data, size_t n); | ^~~~~~ ./snappy-fork/snappy-sinksource.h:164:17: error: ‘virtual char* snappy::UncheckedByteArraySink::GetAppendBuffer(size_t, char*)’ can be marked override [-Werror=suggest-override] 164 | virtual char* GetAppendBuffer(size_t len, char* scratch); | ^~~~~~~~~~~~~~~ ./snappy-fork/snappy-sinksource.h:165:17: error: ‘virtual char* snappy::UncheckedByteArraySink::GetAppendBufferVariable(size_t, size_t, char*, size_t, size_t*)’ can be marked override [-Werror=suggest-override] 165 | virtual char* GetAppendBufferVariable( | ^~~~~~~~~~~~~~~~~~~~~~~ ./snappy-fork/snappy-sinksource.h:168:16: error: ‘virtual void snappy::UncheckedByteArraySink::AppendAndTakeOwnership(char*, size_t, void (*)(void*, const char*, size_t), void*)’ can be marked override [-Werror=suggest-override] 168 | virtual void AppendAndTakeOwnership( | ^~~~~~~~~~~~~~~~~~~~~~ cc1plus: all warnings being treated as errors ``` This PR adds the missing override specifiers to the sink implementations, so compilation works fine again. Tested it with g++-9.3 and g++-10.2. Compatibility note: Override specifiers were introduced with C++11, which Snappy seems to effectively require, at least according to its CMakeLists.txt file and due to the usage of some C++11-only STL types in its tests.
2020-10-29 20:50:50 +00:00
size_t scratch_size, size_t* allocated_size) override;
void AppendAndTakeOwnership(
Add support for Uncompress(source, sink). Various changes to allow Uncompress(source, sink) to get the same performance as the different variants of Uncompress to Cord/DataBuffer/String/FlatBuffer. Changes to efficiently support Uncompress(source, sink) -------- a) For strings - we add support to StringByteSink to do GetAppendBuffer so we can write to it without copying. b) For flat array buffers, we do GetAppendBuffer and see if we can get a full buffer. With the above changes we get performance with ByteSource/ByteSink that is very close to directly using flat arrays and strings. We add various benchmark cases to demonstrate that. Orthogonal change ------------------ Add support for TryFastAppend() for SnappyScatteredWriter. Benchmark results are below CPU: Intel Core2 dL1:32KB dL2:4096KB Benchmark Time(ns) CPU(ns) Iterations ----------------------------------------------------- BM_UFlat/0 109065 108996 6410 896.0MB/s html BM_UFlat/1 1012175 1012343 691 661.4MB/s urls BM_UFlat/2 26775 26771 26149 4.4GB/s jpg BM_UFlat/3 48947 48940 14363 1.8GB/s pdf BM_UFlat/4 441029 440835 1589 886.1MB/s html4 BM_UFlat/5 39861 39880 17823 588.3MB/s cp BM_UFlat/6 18315 18300 38126 581.1MB/s c BM_UFlat/7 5254 5254 100000 675.4MB/s lsp BM_UFlat/8 1568060 1567376 447 626.6MB/s xls BM_UFlat/9 337512 337734 2073 429.5MB/s txt1 BM_UFlat/10 287269 287054 2434 415.9MB/s txt2 BM_UFlat/11 890098 890219 787 457.2MB/s txt3 BM_UFlat/12 1186593 1186863 590 387.2MB/s txt4 BM_UFlat/13 573927 573318 1000 853.7MB/s bin BM_UFlat/14 64250 64294 10000 567.2MB/s sum BM_UFlat/15 7301 7300 96153 552.2MB/s man BM_UFlat/16 109617 109636 6375 1031.5MB/s pb BM_UFlat/17 364438 364497 1921 482.3MB/s gaviota BM_UFlatSink/0 108518 108465 6450 900.4MB/s html BM_UFlatSink/1 991952 991997 705 675.0MB/s urls BM_UFlatSink/2 26815 26798 26065 4.4GB/s jpg BM_UFlatSink/3 49127 49122 14255 1.8GB/s pdf BM_UFlatSink/4 436674 436731 1604 894.4MB/s html4 BM_UFlatSink/5 39738 39733 17345 590.5MB/s cp BM_UFlatSink/6 18413 18416 37962 577.4MB/s c BM_UFlatSink/7 5677 5676 100000 625.2MB/s lsp BM_UFlatSink/8 1552175 1551026 451 633.2MB/s xls BM_UFlatSink/9 338526 338489 2065 428.5MB/s txt1 BM_UFlatSink/10 289387 289307 2420 412.6MB/s txt2 BM_UFlatSink/11 893803 893706 783 455.4MB/s txt3 BM_UFlatSink/12 1195919 1195459 586 384.4MB/s txt4 BM_UFlatSink/13 559637 559779 1000 874.3MB/s bin BM_UFlatSink/14 65073 65094 10000 560.2MB/s sum BM_UFlatSink/15 7618 7614 92823 529.5MB/s man BM_UFlatSink/16 110085 110121 6352 1027.0MB/s pb BM_UFlatSink/17 369196 368915 1896 476.5MB/s gaviota BM_UValidate/0 46954 46957 14899 2.0GB/s html BM_UValidate/1 500621 500868 1000 1.3GB/s urls BM_UValidate/2 283 283 2481447 417.2GB/s jpg BM_UValidate/3 16230 16228 43137 5.4GB/s pdf BM_UValidate/4 189129 189193 3701 2.0GB/s html4 A=uday R=sanjay
2015-06-22 14:03:28 +00:00
char* bytes, size_t n, void (*deleter)(void*, const char*, size_t),
fix compile warnings due to missing override specifiers When building Snappy with compiler option `-Wsuggest-override` set via the CMAKE_CXX_FLAGS, compilation produces warnings in `snappy-sinksource.h`: ``` In file included from ./snappy-fork/snappy-sinksource.cc:32: ./snappy-fork/snappy-sinksource.h:150:18: error: ‘virtual size_t snappy::ByteArraySource::Available() const’ can be marked override [-Werror=suggest-override] 150 | virtual size_t Available() const; | ^~~~~~~~~ ./snappy-fork/snappy-sinksource.h:151:23: error: ‘virtual const char* snappy::ByteArraySource::Peek(size_t*)’ can be marked override [-Werror=suggest-override] 151 | virtual const char* Peek(size_t* len); | ^~~~ ./snappy-fork/snappy-sinksource.h:152:16: error: ‘virtual void snappy::ByteArraySource::Skip(size_t)’ can be marked override [-Werror=suggest-override] 152 | virtual void Skip(size_t n); | ^~~~ ./snappy-fork/snappy-sinksource.h:163:16: error: ‘virtual void snappy::UncheckedByteArraySink::Append(const char*, size_t)’ can be marked override [-Werror=suggest-override] 163 | virtual void Append(const char* data, size_t n); | ^~~~~~ ./snappy-fork/snappy-sinksource.h:164:17: error: ‘virtual char* snappy::UncheckedByteArraySink::GetAppendBuffer(size_t, char*)’ can be marked override [-Werror=suggest-override] 164 | virtual char* GetAppendBuffer(size_t len, char* scratch); | ^~~~~~~~~~~~~~~ ./snappy-fork/snappy-sinksource.h:165:17: error: ‘virtual char* snappy::UncheckedByteArraySink::GetAppendBufferVariable(size_t, size_t, char*, size_t, size_t*)’ can be marked override [-Werror=suggest-override] 165 | virtual char* GetAppendBufferVariable( | ^~~~~~~~~~~~~~~~~~~~~~~ ./snappy-fork/snappy-sinksource.h:168:16: error: ‘virtual void snappy::UncheckedByteArraySink::AppendAndTakeOwnership(char*, size_t, void (*)(void*, const char*, size_t), void*)’ can be marked override [-Werror=suggest-override] 168 | virtual void AppendAndTakeOwnership( | ^~~~~~~~~~~~~~~~~~~~~~ cc1plus: all warnings being treated as errors ``` This PR adds the missing override specifiers to the sink implementations, so compilation works fine again. Tested it with g++-9.3 and g++-10.2. Compatibility note: Override specifiers were introduced with C++11, which Snappy seems to effectively require, at least according to its CMakeLists.txt file and due to the usage of some C++11-only STL types in its tests.
2020-10-29 20:50:50 +00:00
void *deleter_arg) override;
// Return the current output pointer so that a caller can see how
// many bytes were produced.
// Note: this is not a Sink method.
char* CurrentDestination() const { return dest_; }
private:
char* dest_;
};
} // namespace snappy
#endif // THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_