From 0ce9b3a22d366f664ac21cfb742966b8683cebec Mon Sep 17 00:00:00 2001 From: Zhichao Cao Date: Wed, 23 Sep 2020 19:00:30 -0700 Subject: [PATCH] Add AppendWithVerify and PositionedAppendWithVerify to Env and FileSystem (#7419) Summary: Add new AppendWithVerify and PositionedAppendWithVerify APIs to Env and FileSystem to bring the data verification information (data checksum information) from upper layer (e.g., WritableFileWriter) to the storage layer. This PR only include the API definition, no functional codes are added to unblock other developers which depend on these APIs. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7419 Test Plan: make -j32 Reviewed By: pdillinger Differential Revision: D23883196 Pulled By: zhichao-cao fbshipit-source-id: 94676c26bc56144cc32e3661f84f21eccd790411 --- env/composite_env_wrapper.h | 11 +++++++++ env/file_system_tracer.h | 11 +++++++++ env/io_posix.h | 16 +++++++++++++ include/rocksdb/file_system.h | 41 ++++++++++++++++++++++++++++++++++ utilities/fault_injection_fs.h | 11 +++++++++ 5 files changed, 90 insertions(+) diff --git a/env/composite_env_wrapper.h b/env/composite_env_wrapper.h index f12657a04b..7a0da5c3e2 100644 --- a/env/composite_env_wrapper.h +++ b/env/composite_env_wrapper.h @@ -727,11 +727,22 @@ class LegacyWritableFileWrapper : public FSWritableFile { IODebugContext* /*dbg*/) override { return status_to_io_status(target_->Append(data)); } + IOStatus Append(const Slice& data, const IOOptions& /*options*/, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Append(data)); + } IOStatus PositionedAppend(const Slice& data, uint64_t offset, const IOOptions& /*options*/, IODebugContext* /*dbg*/) override { return status_to_io_status(target_->PositionedAppend(data, offset)); } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& /*options*/, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->PositionedAppend(data, offset)); + } IOStatus Truncate(uint64_t size, const IOOptions& /*options*/, IODebugContext* /*dbg*/) override { return status_to_io_status(target_->Truncate(size)); diff --git a/env/file_system_tracer.h b/env/file_system_tracer.h index f18d4a1ff9..4f37cbb0f2 100644 --- a/env/file_system_tracer.h +++ b/env/file_system_tracer.h @@ -244,10 +244,21 @@ class FSWritableFileTracingWrapper : public FSWritableFileWrapper { IOStatus Append(const Slice& data, const IOOptions& options, IODebugContext* dbg) override; + IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* dbg) override { + return Append(data, options, dbg); + } IOStatus PositionedAppend(const Slice& data, uint64_t offset, const IOOptions& options, IODebugContext* dbg) override; + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* dbg) override { + return PositionedAppend(data, offset, options, dbg); + } IOStatus Truncate(uint64_t size, const IOOptions& options, IODebugContext* dbg) override; diff --git a/env/io_posix.h b/env/io_posix.h index 61d3cda9dd..2d8e83c9d7 100644 --- a/env/io_posix.h +++ b/env/io_posix.h @@ -242,9 +242,20 @@ class PosixWritableFile : public FSWritableFile { virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override; virtual IOStatus Append(const Slice& data, const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Append(const Slice& data, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return Append(data, opts, dbg); + } virtual IOStatus PositionedAppend(const Slice& data, uint64_t offset, const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus PositionedAppend( + const Slice& data, uint64_t offset, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return PositionedAppend(data, offset, opts, dbg); + } virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override; virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override; virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override; @@ -331,6 +342,11 @@ class PosixMmapFile : public FSWritableFile { virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override; virtual IOStatus Append(const Slice& data, const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Append(const Slice& data, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return Append(data, opts, dbg); + } virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override; virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override; virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override; diff --git a/include/rocksdb/file_system.h b/include/rocksdb/file_system.h index 76009c2bb2..3683491c1b 100644 --- a/include/rocksdb/file_system.h +++ b/include/rocksdb/file_system.h @@ -702,6 +702,13 @@ class FSRandomAccessFile { // RandomAccessFileWrapper too. }; +// A data structure brings the data verification information, which is +// used togther with data being written to a file. +struct DataVerificationInfo { + // checksum of the data being written. + Slice checksum; +}; + // A file abstraction for sequential writing. The implementation // must provide buffering since callers may append small fragments // at a time to the file. @@ -729,6 +736,16 @@ class FSWritableFile { virtual IOStatus Append(const Slice& data, const IOOptions& options, IODebugContext* dbg) = 0; + // EXPERIMENTAL / CURRENTLY UNUSED + // Append data with verification information + // Note that this API change is experimental and it might be changed in + // the future. Currently, RocksDB does not use this API. + virtual IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) { + return Append(data, options, dbg); + } + // PositionedAppend data to the specified offset. The new EOF after append // must be larger than the previous EOF. This is to be used when writes are // not backed by OS buffers and hence has to always start from the start of @@ -756,6 +773,18 @@ class FSWritableFile { return IOStatus::NotSupported(); } + // EXPERIMENTAL / CURRENTLY UNUSED + // PositionedAppend data with verification information. + // Note that this API change is experimental and it might be changed in + // the future. Currently, RocksDB does not use this API. + virtual IOStatus PositionedAppend( + const Slice& /* data */, uint64_t /* offset */, + const IOOptions& /*options*/, + const DataVerificationInfo& /* verification_info */, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported(); + } + // Truncate is necessary to trim the file to the correct size // before closing. It is not always possible to keep track of the file // size due to whole pages writes. The behavior is undefined if called @@ -1286,11 +1315,23 @@ class FSWritableFileWrapper : public FSWritableFile { IODebugContext* dbg) override { return target_->Append(data, options, dbg); } + IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override { + return target_->Append(data, options, verification_info, dbg); + } IOStatus PositionedAppend(const Slice& data, uint64_t offset, const IOOptions& options, IODebugContext* dbg) override { return target_->PositionedAppend(data, offset, options, dbg); } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override { + return target_->PositionedAppend(data, offset, options, verification_info, + dbg); + } IOStatus Truncate(uint64_t size, const IOOptions& options, IODebugContext* dbg) override { return target_->Truncate(size, options, dbg); diff --git a/utilities/fault_injection_fs.h b/utilities/fault_injection_fs.h index d856cba5db..4c4a1612b8 100644 --- a/utilities/fault_injection_fs.h +++ b/utilities/fault_injection_fs.h @@ -64,6 +64,11 @@ class TestFSWritableFile : public FSWritableFile { virtual ~TestFSWritableFile(); virtual IOStatus Append(const Slice& data, const IOOptions&, IODebugContext*) override; + virtual IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* dbg) override { + return Append(data, options, dbg); + } virtual IOStatus Truncate(uint64_t size, const IOOptions& options, IODebugContext* dbg) override { return target_->Truncate(size, options, dbg); @@ -78,6 +83,12 @@ class TestFSWritableFile : public FSWritableFile { IODebugContext* dbg) override { return target_->PositionedAppend(data, offset, options, dbg); } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* dbg) override { + return PositionedAppend(data, offset, options, dbg); + } virtual size_t GetRequiredBufferAlignment() const override { return target_->GetRequiredBufferAlignment(); }