dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make FileSystem a Customizable Class (#8649) 

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8649

Reviewed By: zhichao-cao

Differential Revision: D32036059

Pulled By: mrambacher

fbshipit-source-id: 4f1e7557ecac52eb849b83ae02b8d7d232112295
This commit is contained in:
mrambacher 2021-11-02 09:06:02 -07:00 committed by Facebook GitHub Bot
parent cfc57f55b5
commit f72c834eab
24 changed files with 874 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
HISTORY.md
View File

@ -15,6 +15,9 @@
### Public Interface Change
* When options.ttl is used with leveled compaction with compactinon priority kMinOverlappingRatio, files exceeding half of TTL value will be prioritized more, so that by the time TTL is reached, fewer extra compactions will be scheduled to clear them up. At the same time, when compacting files with data older than half of TTL, output files may be cut off based on those files' boundaries, in order for the early TTL compaction to work properly.
### Public API change
* Made FileSystem extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
## 6.26.0 (2021-10-20)
### Bug Fixes
* Fixes a bug in directed IO mode when calling MultiGet() for blobs in the same blob file. The bug is caused by not sorting the blob read requests by file offsets.

3
db/db_basic_test.cc
View File

@ -3296,6 +3296,9 @@ class DeadlineFS : public FileSystemWrapper {
ignore_deadline_(false),
error_on_delay_(error_on_delay) {}
static const char* kClassName() { return "DeadlineFileSystem"; }
const char* Name() const override { return kClassName(); }
IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& opts,
std::unique_ptr<FSRandomAccessFile>* result,

3
env/env.cc vendored
View File

@ -324,7 +324,8 @@ class LegacyFileSystemWrapper : public FileSystem {
explicit LegacyFileSystemWrapper(Env* t) : target_(t) {}
~LegacyFileSystemWrapper() override {}
const char* Name() const override { return "Legacy File System"; }
static const char* kClassName() { return "LegacyFileSystem"; }
const char* Name() const override { return kClassName(); }
// Return the target to which this Env forwards all calls
Env* target() const { return target_; }

127
env/env_chroot.cc vendored
View File

@ -13,20 +13,35 @@
#include "env/composite_env_wrapper.h"
#include "env/fs_remap.h"
#include "rocksdb/utilities/options_type.h"
#include "util/string_util.h" // errnoStr
namespace ROCKSDB_NAMESPACE {
namespace {
class ChrootFileSystem : public RemapFileSystem {
public:
ChrootFileSystem(const std::shared_ptr<FileSystem>& base,
const std::string& chroot_dir)
: RemapFileSystem(base) {
static std::unordered_map<std::string, OptionTypeInfo> chroot_fs_type_info = {
{"chroot_dir", {0, OptionType::kString}}};
} // namespace
ChrootFileSystem::ChrootFileSystem(const std::shared_ptr<FileSystem>& base,
const std::string& chroot_dir)
: RemapFileSystem(base), chroot_dir_(chroot_dir) {
RegisterOptions("chroot_dir", &chroot_dir_, &chroot_fs_type_info);
}
Status ChrootFileSystem::PrepareOptions(const ConfigOptions& options) {
Status s = FileSystemWrapper::PrepareOptions(options);
if (!s.ok()) {
return s;
} else if (chroot_dir_.empty()) {
s = Status::InvalidArgument("ChRootFileSystem requires a chroot dir");
} else {
s = target_->FileExists(chroot_dir_, IOOptions(), nullptr);
}
if (s.ok()) {
#if defined(OS_AIX)
char resolvedName[PATH_MAX];
char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName);
char* real_chroot_dir = realpath(chroot_dir_.c_str(), resolvedName);
#else
char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr);
char* real_chroot_dir = realpath(chroot_dir_.c_str(), nullptr);
#endif
// chroot_dir must exist so realpath() returns non-nullptr.
assert(real_chroot_dir != nullptr);
@ -35,32 +50,32 @@ class ChrootFileSystem : public RemapFileSystem {
free(real_chroot_dir);
#endif
}
return s;
}
const char* Name() const override { return "ChrootFS"; }
IOStatus ChrootFileSystem::GetTestDirectory(const IOOptions& options,
std::string* path,
IODebugContext* dbg) {
// Adapted from PosixEnv's implementation since it doesn't provide a way to
// create directory in the chroot.
char buf[256];
snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast<int>(geteuid()));
*path = buf;
IOStatus GetTestDirectory(const IOOptions& options, std::string* path,
IODebugContext* dbg) override {
// Adapted from PosixEnv's implementation since it doesn't provide a way to
// create directory in the chroot.
char buf[256];
snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast<int>(geteuid()));
*path = buf;
// Directory may already exist, so ignore return
return CreateDirIfMissing(*path, options, dbg);
}
// Directory may already exist, so ignore return
return CreateDirIfMissing(*path, options, dbg);
}
protected:
// Returns status and expanded absolute path including the chroot directory.
// Checks whether the provided path breaks out of the chroot. If it returns
// non-OK status, the returned path should not be used.
std::pair<IOStatus, std::string> EncodePath(
const std::string& path) override {
if (path.empty() || path[0] != '/') {
return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""};
}
std::pair<IOStatus, std::string> res;
res.second = chroot_dir_ + path;
std::pair<IOStatus, std::string> ChrootFileSystem::EncodePath(
const std::string& path) {
if (path.empty() || path[0] != '/') {
return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""};
}
std::pair<IOStatus, std::string> res;
res.second = chroot_dir_ + path;
#if defined(OS_AIX)
char resolvedName[PATH_MAX];
char* normalized_path = realpath(res.second.c_str(), resolvedName);
@ -81,47 +96,51 @@ class ChrootFileSystem : public RemapFileSystem {
free(normalized_path);
#endif
return res;
}
}
// Similar to EncodePath() except assumes the basename in the path hasn't been
// created yet.
std::pair<IOStatus, std::string> EncodePathWithNewBasename(
const std::string& path) override {
if (path.empty() || path[0] != '/') {
return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""};
}
// Basename may be followed by trailing slashes
size_t final_idx = path.find_last_not_of('/');
if (final_idx == std::string::npos) {
// It's only slashes so no basename to extract
return EncodePath(path);
}
// Pull off the basename temporarily since realname(3) (used by
// EncodePath()) requires a path that exists
size_t base_sep = path.rfind('/', final_idx);
auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1));
status_and_enc_path.second.append(path.substr(base_sep + 1));
return status_and_enc_path;
std::pair<IOStatus, std::string> ChrootFileSystem::EncodePathWithNewBasename(
const std::string& path) {
if (path.empty() || path[0] != '/') {
return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""};
}
// Basename may be followed by trailing slashes
size_t final_idx = path.find_last_not_of('/');
if (final_idx == std::string::npos) {
// It's only slashes so no basename to extract
return EncodePath(path);
}
private:
std::string chroot_dir_;
};
} // namespace
// Pull off the basename temporarily since realname(3) (used by
// EncodePath()) requires a path that exists
size_t base_sep = path.rfind('/', final_idx);
auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1));
status_and_enc_path.second.append(path.substr(base_sep + 1));
return status_and_enc_path;
}
std::shared_ptr<FileSystem> NewChrootFileSystem(
const std::shared_ptr<FileSystem>& base, const std::string& chroot_dir) {
return std::make_shared<ChrootFileSystem>(base, chroot_dir);
auto chroot_fs = std::make_shared<ChrootFileSystem>(base, chroot_dir);
Status s = chroot_fs->PrepareOptions(ConfigOptions());
if (s.ok()) {
return chroot_fs;
} else {
return nullptr;
}
}
Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) {
if (!base_env->FileExists(chroot_dir).ok()) {
return nullptr;
}
std::shared_ptr<FileSystem> chroot_fs =
NewChrootFileSystem(base_env->GetFileSystem(), chroot_dir);
return new CompositeEnvWrapper(base_env, chroot_fs);
auto chroot_fs = NewChrootFileSystem(base_env->GetFileSystem(), chroot_dir);
if (chroot_fs != nullptr) {
return new CompositeEnvWrapper(base_env, chroot_fs);
} else {
return nullptr;
}
}
} // namespace ROCKSDB_NAMESPACE

32
env/env_chroot.h vendored
View File

@ -9,9 +9,37 @@
#include <string>
#include "rocksdb/env.h"
#include "env/fs_remap.h"
#include "rocksdb/file_system.h"
namespace ROCKSDB_NAMESPACE {
class ChrootFileSystem : public RemapFileSystem {
public:
ChrootFileSystem(const std::shared_ptr<FileSystem>& base,
const std::string& chroot_dir);
static const char* kClassName() { return "ChrootFS"; }
const char* Name() const override { return kClassName(); }
IOStatus GetTestDirectory(const IOOptions& options, std::string* path,
IODebugContext* dbg) override;
Status PrepareOptions(const ConfigOptions& options) override;
protected:
// Returns status and expanded absolute path including the chroot directory.
// Checks whether the provided path breaks out of the chroot. If it returns
// non-OK status, the returned path should not be used.
std::pair<IOStatus, std::string> EncodePath(const std::string& path) override;
// Similar to EncodePath() except assumes the basename in the path hasn't been
// created yet.
std::pair<IOStatus, std::string> EncodePathWithNewBasename(
const std::string& path) override;
private:
std::string chroot_dir_;
};
// Returns an Env that translates paths such that the root directory appears to
// be chroot_dir. chroot_dir should refer to an existing directory.
@ -19,6 +47,8 @@ namespace ROCKSDB_NAMESPACE {
// This class has not been fully analyzed for providing strong security
// guarantees.
Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir);
std::shared_ptr<FileSystem> NewChrootFileSystem(
const std::shared_ptr<FileSystem>& base, const std::string& chroot_dir);
} // namespace ROCKSDB_NAMESPACE

32
env/env_encryption.cc vendored
View File

@ -438,11 +438,20 @@ IOStatus EncryptedRandomRWFile::Close(const IOOptions& options,
}
namespace {
static std::unordered_map<std::string, OptionTypeInfo> encrypted_fs_type_info =
{
{"provider",
OptionTypeInfo::AsCustomSharedPtr<EncryptionProvider>(
0 /* No offset, whole struct*/, OptionVerificationType::kByName,
OptionTypeFlags::kNone)},
};
// EncryptedFileSystemImpl implements an FileSystemWrapper that adds encryption
// to files stored on disk.
class EncryptedFileSystemImpl : public EncryptedFileSystem {
public:
const char* Name() const override { return "EncryptedFS"; }
const char* Name() const override {
return EncryptedFileSystem::kClassName();
}
// Returns the raw encryption provider that should be used to write the input
// encrypted file. If there is no such provider, NotFound is returned.
IOStatus GetWritableProvider(const std::string& /*fname*/,
@ -664,6 +673,7 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
const std::shared_ptr<EncryptionProvider>& provider)
: EncryptedFileSystem(base) {
provider_ = provider;
RegisterOptions("EncryptionProvider", &provider_, &encrypted_fs_type_info);
}
Status AddCipher(const std::string& descriptor, const char* cipher,
@ -910,10 +920,28 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
};
} // namespace
Status NewEncryptedFileSystemImpl(
const std::shared_ptr<FileSystem>& base,
const std::shared_ptr<EncryptionProvider>& provider,
std::unique_ptr<FileSystem>* result) {
result->reset(new EncryptedFileSystemImpl(base, provider));
return Status::OK();
}
std::shared_ptr<FileSystem> NewEncryptedFS(
const std::shared_ptr<FileSystem>& base,
const std::shared_ptr<EncryptionProvider>& provider) {
return std::make_shared<EncryptedFileSystemImpl>(base, provider);
std::unique_ptr<FileSystem> efs;
Status s = NewEncryptedFileSystemImpl(base, provider, &efs);
if (s.ok()) {
s = efs->PrepareOptions(ConfigOptions());
}
if (s.ok()) {
std::shared_ptr<FileSystem> result(efs.release());
return result;
} else {
return nullptr;
}
}
// Returns an Env that encrypts data when stored on disk and decrypts data when
// read from disk.

6
env/env_encryption_ctr.h vendored
View File

@ -105,6 +105,12 @@ class CTREncryptionProvider : public EncryptionProvider {
uint64_t initialCounter, const Slice& iv, const Slice& prefix,
std::unique_ptr<BlockAccessCipherStream>* result);
};
Status NewEncryptedFileSystemImpl(
const std::shared_ptr<FileSystem>& base_fs,
const std::shared_ptr<EncryptionProvider>& provider,
std::unique_ptr<FileSystem>* fs);
} // namespace ROCKSDB_NAMESPACE
#endif // !defined(ROCKSDB_LITE)

171
env/env_test.cc vendored
View File

@ -39,6 +39,7 @@
#include "env/emulated_clock.h"
#include "env/env_chroot.h"
#include "env/env_encryption_ctr.h"
#include "env/fs_readonly.h"
#include "env/unique_id_gen.h"
#include "logging/log_buffer.h"
#include "logging/logging.h"
@ -59,6 +60,7 @@
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/string_util.h"
#include "utilities/env_timed.h"
#include "utilities/fault_injection_env.h"
#include "utilities/fault_injection_fs.h"
@ -2233,14 +2235,17 @@ INSTANTIATE_TEST_CASE_P(DefaultEnvWithDirectIO, EnvPosixTestWithParam,
#endif // !defined(ROCKSDB_LITE)
#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
static std::unique_ptr<Env> chroot_env(
NewChrootEnv(Env::Default(), test::TmpDir(Env::Default())));
INSTANTIATE_TEST_CASE_P(
ChrootEnvWithoutDirectIO, EnvPosixTestWithParam,
::testing::Values(std::pair<Env*, bool>(chroot_env.get(), false)));
INSTANTIATE_TEST_CASE_P(
ChrootEnvWithDirectIO, EnvPosixTestWithParam,
::testing::Values(std::pair<Env*, bool>(chroot_env.get(), true)));
static Env* GetChrootEnv() {
static std::unique_ptr<Env> chroot_env(
NewChrootEnv(Env::Default(), test::TmpDir(Env::Default())));
return chroot_env.get();
}
INSTANTIATE_TEST_CASE_P(ChrootEnvWithoutDirectIO, EnvPosixTestWithParam,
::testing::Values(std::pair<Env*, bool>(GetChrootEnv(),
false)));
INSTANTIATE_TEST_CASE_P(ChrootEnvWithDirectIO, EnvPosixTestWithParam,
::testing::Values(std::pair<Env*, bool>(GetChrootEnv(),
true)));
#endif // !defined(ROCKSDB_LITE) && !defined(OS_WIN)
class EnvFSTestWithParam
@ -2492,7 +2497,6 @@ TEST_F(CreateEnvTest, CreateMockSystemClock) {
guard->reset(new MockSystemClock(nullptr));
return guard->get();
});
ASSERT_OK(SystemClock::CreateFromString(
config_options_, EmulatedSystemClock::kClassName(), &mock));
ASSERT_NE(mock, nullptr);
@ -2518,6 +2522,154 @@ TEST_F(CreateEnvTest, CreateMockSystemClock) {
ASSERT_OK(SystemClock::CreateFromString(
config_options_, EmulatedSystemClock::kClassName(), &mock));
}
TEST_F(CreateEnvTest, CreateReadOnlyFileSystem) {
std::shared_ptr<FileSystem> fs, copy;
ASSERT_OK(FileSystem::CreateFromString(
config_options_, ReadOnlyFileSystem::kClassName(), &fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), ReadOnlyFileSystem::kClassName());
ASSERT_EQ(fs->Inner(), FileSystem::Default().get());
std::string opts_str = fs->ToString(config_options_);
std::string mismatch;
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(FileSystem::CreateFromString(
config_options_,
std::string("id=") + ReadOnlyFileSystem::kClassName() +
"; target=" + TimedFileSystem::kClassName(),
&fs));
ASSERT_NE(fs, nullptr);
opts_str = fs->ToString(config_options_);
ASSERT_STREQ(fs->Name(), ReadOnlyFileSystem::kClassName());
ASSERT_NE(fs->Inner(), nullptr);
ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName());
ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get());
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
}
TEST_F(CreateEnvTest, CreateTimedFileSystem) {
std::shared_ptr<FileSystem> fs, copy;
ASSERT_OK(FileSystem::CreateFromString(config_options_,
TimedFileSystem::kClassName(), &fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName());
ASSERT_EQ(fs->Inner(), FileSystem::Default().get());
std::string opts_str = fs->ToString(config_options_);
std::string mismatch;
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(FileSystem::CreateFromString(
config_options_,
std::string("id=") + TimedFileSystem::kClassName() +
"; target=" + ReadOnlyFileSystem::kClassName(),
&fs));
ASSERT_NE(fs, nullptr);
opts_str = fs->ToString(config_options_);
ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName());
ASSERT_NE(fs->Inner(), nullptr);
ASSERT_STREQ(fs->Inner()->Name(), ReadOnlyFileSystem::kClassName());
ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get());
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
}
#ifndef OS_WIN
TEST_F(CreateEnvTest, CreateChrootFileSystem) {
std::shared_ptr<FileSystem> fs, copy;
auto tmp_dir = test::TmpDir(Env::Default());
// The Chroot FileSystem has a required "chroot_dir" option.
ASSERT_NOK(FileSystem::CreateFromString(config_options_,
ChrootFileSystem::kClassName(), &fs));
// ChrootFileSystem fails with an invalid directory
ASSERT_NOK(FileSystem::CreateFromString(
config_options_,
std::string("chroot_dir=/No/Such/Directory; id=") +
ChrootFileSystem::kClassName(),
&fs));
std::string chroot_opts = std::string("chroot_dir=") + tmp_dir +
std::string("; id=") +
ChrootFileSystem::kClassName();
// Create a valid ChrootFileSystem with an inner Default
ASSERT_OK(FileSystem::CreateFromString(config_options_, chroot_opts, &fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), ChrootFileSystem::kClassName());
ASSERT_EQ(fs->Inner(), FileSystem::Default().get());
std::string opts_str = fs->ToString(config_options_);
std::string mismatch;
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
// Create a valid ChrootFileSystem with an inner TimedFileSystem
ASSERT_OK(FileSystem::CreateFromString(
config_options_,
chroot_opts + "; target=" + TimedFileSystem::kClassName(), &fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), ChrootFileSystem::kClassName());
ASSERT_NE(fs->Inner(), nullptr);
ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName());
ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get());
opts_str = fs->ToString(config_options_);
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
// Create a TimedFileSystem with an inner ChrootFileSystem
ASSERT_OK(FileSystem::CreateFromString(
config_options_,
"target={" + chroot_opts + "}; id=" + TimedFileSystem::kClassName(),
&fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName());
ASSERT_NE(fs->Inner(), nullptr);
ASSERT_STREQ(fs->Inner()->Name(), ChrootFileSystem::kClassName());
ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get());
opts_str = fs->ToString(config_options_);
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
}
#endif // OS_WIN
TEST_F(CreateEnvTest, CreateEncryptedFileSystem) {
std::shared_ptr<FileSystem> fs, copy;
std::string base_opts =
std::string("provider=1://test; id=") + EncryptedFileSystem::kClassName();
// The EncryptedFileSystem requires a "provider" option.
ASSERT_NOK(FileSystem::CreateFromString(
config_options_, EncryptedFileSystem::kClassName(), &fs));
ASSERT_OK(FileSystem::CreateFromString(config_options_, base_opts, &fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), EncryptedFileSystem::kClassName());
ASSERT_EQ(fs->Inner(), FileSystem::Default().get());
std::string opts_str = fs->ToString(config_options_);
std::string mismatch;
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(FileSystem::CreateFromString(
config_options_, base_opts + "; target=" + TimedFileSystem::kClassName(),
&fs));
ASSERT_NE(fs, nullptr);
ASSERT_STREQ(fs->Name(), EncryptedFileSystem::kClassName());
ASSERT_NE(fs->Inner(), nullptr);
ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName());
ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get());
opts_str = fs->ToString(config_options_);
ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
}
#endif // ROCKSDB_LITE
namespace {
@ -2742,7 +2894,6 @@ TEST_F(EnvTest, FailureToCreateLockFile) {
// Clean up
ASSERT_OK(DestroyDir(env, dir));
}
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {

112
env/file_system.cc vendored
View File

@ -6,9 +6,17 @@
#include "rocksdb/file_system.h"
#include "env/composite_env_wrapper.h"
#include "env/env_chroot.h"
#include "env/env_encryption_ctr.h"
#include "env/fs_readonly.h"
#include "env/mock_env.h"
#include "options/db_options.h"
#include "rocksdb/convenience.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "util/string_util.h"
#include "utilities/env_timed.h"
namespace ROCKSDB_NAMESPACE {
@ -21,19 +29,63 @@ Status FileSystem::Load(const std::string& value,
return CreateFromString(ConfigOptions(), value, result);
}
#ifndef ROCKSDB_LITE
static int RegisterBuiltinFileSystems(ObjectLibrary& library,
const std::string& /*arg*/) {
library.Register<FileSystem>(
TimedFileSystem::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
std::string* /* errmsg */) {
guard->reset(new TimedFileSystem(nullptr));
return guard->get();
});
library.Register<FileSystem>(
ReadOnlyFileSystem::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
std::string* /* errmsg */) {
guard->reset(new ReadOnlyFileSystem(nullptr));
return guard->get();
});
library.Register<FileSystem>(
EncryptedFileSystem::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
std::string* errmsg) {
Status s = NewEncryptedFileSystemImpl(nullptr, nullptr, guard);
if (!s.ok()) {
*errmsg = s.ToString();
}
return guard->get();
});
#ifndef OS_WIN
library.Register<FileSystem>(
ChrootFileSystem::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
std::string* /* errmsg */) {
guard->reset(new ChrootFileSystem(nullptr, ""));
return guard->get();
});
#endif // OS_WIN
size_t num_types;
return static_cast<int>(library.GetFactoryCount(&num_types));
}
#endif // ROCKSDB_LITE
Status FileSystem::CreateFromString(const ConfigOptions& config_options,
const std::string& value,
std::shared_ptr<FileSystem>* result) {
Status s;
auto default_fs = FileSystem::Default();
if (default_fs->IsInstanceOf(value)) {
*result = default_fs;
return Status::OK();
} else {
#ifndef ROCKSDB_LITE
(void)config_options;
s = ObjectRegistry::NewInstance()->NewSharedObject<FileSystem>(value, result);
#else
(void)config_options;
(void)result;
s = Status::NotSupported("Cannot load FileSystem in LITE mode", value);
#endif
return s;
static std::once_flag once;
std::call_once(once, [&]() {
RegisterBuiltinFileSystems(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
return LoadSharedObject<FileSystem>(config_options, value, nullptr, result);
}
}
IOStatus FileSystem::ReuseWritableFile(const std::string& fname,
@ -147,4 +199,46 @@ IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
return s;
}
namespace {
static std::unordered_map<std::string, OptionTypeInfo> fs_wrapper_type_info = {
#ifndef ROCKSDB_LITE
{"target",
OptionTypeInfo::AsCustomSharedPtr<FileSystem>(
0, OptionVerificationType::kByName, OptionTypeFlags::kDontSerialize)},
#endif // ROCKSDB_LITE
};
} // namespace
FileSystemWrapper::FileSystemWrapper(const std::shared_ptr<FileSystem>& t)
: target_(t) {
RegisterOptions("", &target_, &fs_wrapper_type_info);
}
Status FileSystemWrapper::PrepareOptions(const ConfigOptions& options) {
if (target_ == nullptr) {
target_ = FileSystem::Default();
}
return FileSystem::PrepareOptions(options);
}
#ifndef ROCKSDB_LITE
std::string FileSystemWrapper::SerializeOptions(
const ConfigOptions& config_options, const std::string& header) const {
auto parent = FileSystem::SerializeOptions(config_options, "");
if (config_options.IsShallow() || target_ == nullptr ||
target_->IsInstanceOf(FileSystem::kDefaultName())) {
return parent;
} else {
std::string result = header;
if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) {
result.append(OptionTypeInfo::kIdPropName()).append("=");
}
result.append(parent);
if (!EndsWith(result, config_options.delimiter)) {
result.append(config_options.delimiter);
}
result.append("target=").append(target_->ToString(config_options));
return result;
}
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

3
env/file_system_tracer.h vendored
View File

@ -27,6 +27,9 @@ class FileSystemTracingWrapper : public FileSystemWrapper {
~FileSystemTracingWrapper() override {}
static const char* kClassName() { return "FileSystemTracing"; }
const char* Name() const override { return kClassName(); }
IOStatus NewSequentialFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSSequentialFile>* result,

18
env/fs_posix.cc vendored
View File

@ -141,7 +141,9 @@ class PosixFileSystem : public FileSystem {
public:
PosixFileSystem();
const char* Name() const override { return "Posix File System"; }
static const char* kClassName() { return "PosixFileSystem"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const override { return kDefaultName(); }
~PosixFileSystem() override {}
@ -618,7 +620,8 @@ class PosixFileSystem : public FileSystem {
return IOStatus::NotFound();
default:
assert(err == EIO || err == ENOMEM);
return IOStatus::IOError("Unexpected error(" + ToString(err) +
return IOStatus::IOError("Unexpected error(" +
ROCKSDB_NAMESPACE::ToString(err) +
") accessing file `" + fname + "' ");
}
}
@ -819,11 +822,12 @@ class PosixFileSystem : public FileSystem {
errno = ENOLCK;
// Note that the thread ID printed is the same one as the one in
// posix logger, but posix logger prints it hex format.
return IOError("lock hold by current process, acquire time " +
ToString(prev_info.acquire_time) +
" acquiring thread " +
ToString(prev_info.acquiring_thread),
fname, errno);
return IOError(
"lock hold by current process, acquire time " +
ROCKSDB_NAMESPACE::ToString(prev_info.acquire_time) +
" acquiring thread " +
ROCKSDB_NAMESPACE::ToString(prev_info.acquiring_thread),
fname, errno);
}
IOStatus result = IOStatus::OK();

3
env/fs_readonly.h vendored
View File

@ -26,6 +26,9 @@ class ReadOnlyFileSystem : public FileSystemWrapper {
explicit ReadOnlyFileSystem(const std::shared_ptr<FileSystem>& base)
: FileSystemWrapper(base) {}
static const char* kClassName() { return "ReadOnlyFileSystem"; }
const char* Name() const override { return kClassName(); }
IOStatus NewWritableFile(const std::string& /*fname*/,
const FileOptions& /*options*/,
std::unique_ptr<FSWritableFile>* /*result*/,

8
env/fs_remap.h vendored
View File

@ -39,6 +39,14 @@ class RemapFileSystem : public FileSystemWrapper {
public:
// Left abstract:
// const char* Name() const override { ... }
static const char* kClassName() { return "RemapFileSystem"; }
bool IsInstanceOf(const std::string& id) const override {
if (id == kClassName()) {
return true;
} else {
return FileSystemWrapper::IsInstanceOf(id);
}
}
Status RegisterDbPaths(const std::vector<std::string>& paths) override;

147
env/mock_env.cc vendored
View File

@ -565,132 +565,39 @@ class TestMemLogger : public Logger {
}
size_t GetLogFileSize() const override { return log_size_; }
};
} // namespace
class MockFileSystem : public FileSystem {
public:
explicit MockFileSystem(const std::shared_ptr<SystemClock>& clock,
bool supports_direct_io = true)
: system_clock_(clock), supports_direct_io_(supports_direct_io) {
clock_ = system_clock_.get();
MockFileSystem::MockFileSystem(const std::shared_ptr<SystemClock>& clock,
bool supports_direct_io)
: system_clock_(clock), supports_direct_io_(supports_direct_io) {
clock_ = system_clock_.get();
}
MockFileSystem::~MockFileSystem() {
for (auto i = file_map_.begin(); i != file_map_.end(); ++i) {
i->second->Unref();
}
~MockFileSystem() override {
for (auto i = file_map_.begin(); i != file_map_.end(); ++i) {
i->second->Unref();
}
}
IOStatus MockFileSystem::GetAbsolutePath(const std::string& db_path,
const IOOptions& /*options*/,
std::string* output_path,
IODebugContext* /*dbg*/) {
*output_path = NormalizeMockPath(db_path);
if (output_path->at(0) != '/') {
return IOStatus::NotSupported("GetAbsolutePath");
} else {
return IOStatus::OK();
}
}
const char* Name() const override { return "Memory"; }
IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts,
std::unique_ptr<FSSequentialFile>* r,
IODebugContext* dbg) override;
IOStatus NewRandomAccessFile(const std::string& f,
const FileOptions& file_opts,
std::unique_ptr<FSRandomAccessFile>* r,
IODebugContext* dbg) override;
IOStatus NewRandomRWFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* dbg) override;
IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus NewWritableFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus ReopenWritableFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus NewDirectory(const std::string& /*name*/, const IOOptions& io_opts,
std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override;
IOStatus FileExists(const std::string& fname, const IOOptions& /*io_opts*/,
IODebugContext* /*dbg*/) override;
IOStatus GetChildren(const std::string& dir, const IOOptions& options,
std::vector<std::string>* result,
IODebugContext* dbg) override;
IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus Truncate(const std::string& fname, size_t size,
const IOOptions& options, IODebugContext* dbg) override;
IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override;
IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override;
IOStatus GetFileModificationTime(const std::string& fname,
const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override;
IOStatus RenameFile(const std::string& src, const std::string& target,
const IOOptions& options, IODebugContext* dbg) override;
IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override;
IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override;
IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetTestDirectory(const IOOptions& options, std::string* path,
IODebugContext* dbg) override;
IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts,
std::shared_ptr<Logger>* result,
IODebugContext* dbg) override;
// Get full directory name for this db.
IOStatus GetAbsolutePath(const std::string& db_path,
const IOOptions& /*options*/,
std::string* output_path,
IODebugContext* /*dbg*/) override {
*output_path = NormalizeMockPath(db_path);
if (output_path->at(0) != '/') {
return IOStatus::NotSupported("GetAbsolutePath");
} else {
return IOStatus::OK();
}
}
IOStatus IsDirectory(const std::string& /*path*/,
const IOOptions& /*options*/, bool* /*is_dir*/,
IODebugContext* /*dgb*/) override {
return IOStatus::NotSupported("IsDirectory");
std::string MockFileSystem::NormalizeMockPath(const std::string& path) {
std::string p = NormalizePath(path);
if (p.back() == kFilePathSeparator && p.size() > 1) {
p.pop_back();
}
return p;
}
Status CorruptBuffer(const std::string& fname);
private:
bool RenameFileInternal(const std::string& src, const std::string& dest);
void DeleteFileInternal(const std::string& fname);
bool GetChildrenInternal(const std::string& fname,
std::vector<std::string>* results);
std::string NormalizeMockPath(const std::string& path) {
std::string p = NormalizePath(path);
if (p.back() == kFilePathSeparator && p.size() > 1) {
p.pop_back();
}
return p;
}
private:
// Map from filenames to MemFile objects, representing a simple file system.
port::Mutex mutex_;
std::map<std::string, MemFile*> file_map_; // Protected by mutex_.
std::shared_ptr<SystemClock> system_clock_;
SystemClock* clock_;
bool supports_direct_io_;
};
} // Anonymous namespace
// Partial implementation of the FileSystem interface.
IOStatus MockFileSystem::NewSequentialFile(
const std::string& fname, const FileOptions& file_opts,

106
env/mock_env.h vendored
View File

@ -14,11 +14,117 @@
#include <vector>
#include "env/composite_env_wrapper.h"
#include "port/port.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "rocksdb/system_clock.h"
namespace ROCKSDB_NAMESPACE {
class MemFile;
class MockFileSystem : public FileSystem {
public:
explicit MockFileSystem(const std::shared_ptr<SystemClock>& clock,
bool supports_direct_io = true);
~MockFileSystem() override;
static const char* kClassName() { return "MemoryFileSystem"; }
const char* Name() const override { return kClassName(); }
IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts,
std::unique_ptr<FSSequentialFile>* r,
IODebugContext* dbg) override;
IOStatus NewRandomAccessFile(const std::string& f,
const FileOptions& file_opts,
std::unique_ptr<FSRandomAccessFile>* r,
IODebugContext* dbg) override;
IOStatus NewRandomRWFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* dbg) override;
IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus NewWritableFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus ReopenWritableFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus NewDirectory(const std::string& /*name*/, const IOOptions& io_opts,
std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override;
IOStatus FileExists(const std::string& fname, const IOOptions& /*io_opts*/,
IODebugContext* /*dbg*/) override;
IOStatus GetChildren(const std::string& dir, const IOOptions& options,
std::vector<std::string>* result,
IODebugContext* dbg) override;
IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus Truncate(const std::string& fname, size_t size,
const IOOptions& options, IODebugContext* dbg) override;
IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override;
IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override;
IOStatus GetFileModificationTime(const std::string& fname,
const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override;
IOStatus RenameFile(const std::string& src, const std::string& target,
const IOOptions& options, IODebugContext* dbg) override;
IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override;
IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override;
IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetTestDirectory(const IOOptions& options, std::string* path,
IODebugContext* dbg) override;
IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts,
std::shared_ptr<Logger>* result,
IODebugContext* dbg) override;
// Get full directory name for this db.
IOStatus GetAbsolutePath(const std::string& db_path,
const IOOptions& /*options*/,
std::string* output_path,
IODebugContext* /*dbg*/) override;
IOStatus IsDirectory(const std::string& /*path*/,
const IOOptions& /*options*/, bool* /*is_dir*/,
IODebugContext* /*dgb*/) override {
return IOStatus::NotSupported("IsDirectory");
}
Status CorruptBuffer(const std::string& fname);
private:
bool RenameFileInternal(const std::string& src, const std::string& dest);
void DeleteFileInternal(const std::string& fname);
bool GetChildrenInternal(const std::string& fname,
std::vector<std::string>* results);
std::string NormalizeMockPath(const std::string& path);
private:
// Map from filenames to MemFile objects, representing a simple file system.
port::Mutex mutex_;
std::map<std::string, MemFile*> file_map_; // Protected by mutex_.
std::shared_ptr<SystemClock> system_clock_;
SystemClock* clock_;
bool supports_direct_io_;
};
class MockEnv : public CompositeEnvWrapper {
public:
static MockEnv* Create(Env* base);

3
file/prefetch_test.cc
View File

@ -39,6 +39,9 @@ class MockFS : public FileSystemWrapper {
bool support_prefetch)
: FileSystemWrapper(wrapped), support_prefetch_(support_prefetch) {}
static const char* kClassName() { return "MockFS"; }
const char* Name() const override { return kClassName(); }
IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& opts,
std::unique_ptr<FSRandomAccessFile>* result,

11
include/rocksdb/env_encryption.h
View File

@ -24,6 +24,9 @@ struct ConfigOptions;
// read from disk.
Env* NewEncryptedEnv(Env* base_env,
const std::shared_ptr<EncryptionProvider>& provider);
std::shared_ptr<FileSystem> NewEncryptedFS(
const std::shared_ptr<FileSystem>& base_fs,
const std::shared_ptr<EncryptionProvider>& provider);
// BlockAccessCipherStream is the base class for any cipher stream that
// supports random access at block level (without requiring data from other
@ -440,6 +443,14 @@ class EncryptedFileSystem : public FileSystemWrapper {
// otherwise
virtual Status AddCipher(const std::string& descriptor, const char* cipher,
size_t len, bool for_write) = 0;
static const char* kClassName() { return "EncryptedFileSystem"; }
bool IsInstanceOf(const std::string& name) const override {
if (name == kClassName()) {
return true;
} else {
return FileSystemWrapper::IsInstanceOf(name);
}
}
};
} // namespace ROCKSDB_NAMESPACE

20
include/rocksdb/file_system.h
View File

@ -28,6 +28,7 @@
#include <unordered_map>
#include <vector>
#include "rocksdb/customizable.h"
#include "rocksdb/env.h"
#include "rocksdb/io_status.h"
#include "rocksdb/options.h"
@ -207,7 +208,8 @@ struct IODebugContext {
// retryable.
// NewCompositeEnv can be used to create an Env with a custom FileSystem for
// DBOptions::env.
class FileSystem {
class FileSystem : public Customizable {
public:
FileSystem();
@ -216,9 +218,8 @@ class FileSystem {
virtual ~FileSystem();
virtual const char* Name() const = 0;
static const char* Type() { return "FileSystem"; }
static const char* kDefaultName() { return "DefaultFileSystem"; }
// Loads the FileSystem specified by the input value into the result
// The CreateFromString alternative should be used; this method may be
@ -1147,12 +1148,9 @@ class FSDirectory {
class FileSystemWrapper : public FileSystem {
public:
// Initialize an EnvWrapper that delegates all calls to *t
explicit FileSystemWrapper(const std::shared_ptr<FileSystem>& t)
: target_(t) {}
explicit FileSystemWrapper(const std::shared_ptr<FileSystem>& t);
~FileSystemWrapper() override {}
const char* Name() const override { return target_->Name(); }
// Return the target to which this Env forwards all calls
FileSystem* target() const { return target_.get(); }
@ -1343,7 +1341,13 @@ class FileSystemWrapper : public FileSystem {
return target_->IsDirectory(path, options, is_dir, dbg);
}
private:
const Customizable* Inner() const override { return target_.get(); }
Status PrepareOptions(const ConfigOptions& options) override;
#ifndef ROCKSDB_LITE
std::string SerializeOptions(const ConfigOptions& config_options,
const std::string& header) const override;
#endif // ROCKSDB_LITE
protected:
std::shared_ptr<FileSystem> target_;
};

37
options/customizable_test.cc
View File

@ -1299,6 +1299,17 @@ class MockCipher : public BlockCipher {
Status Encrypt(char* /*data*/) override { return Status::NotSupported(); }
Status Decrypt(char* data) override { return Encrypt(data); }
};
#endif // ROCKSDB_LITE
class DummyFileSystem : public FileSystemWrapper {
public:
explicit DummyFileSystem(const std::shared_ptr<FileSystem>& t)
: FileSystemWrapper(t) {}
static const char* kClassName() { return "DummyFileSystem"; }
const char* Name() const override { return kClassName(); }
};
#ifndef ROCKSDB_LITE
#endif // ROCKSDB_LITE
@ -1406,6 +1417,14 @@ static int RegisterLocalObjects(ObjectLibrary& library,
return guard->get();
});
library.Register<FileSystem>(
DummyFileSystem::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
std::string* /* errmsg */) {
guard->reset(new DummyFileSystem(nullptr));
return guard->get();
});
library.Register<SstPartitionerFactory>(
MockSstPartitionerFactory::kClassName(),
[](const std::string& /*uri*/,
@ -1495,6 +1514,24 @@ TEST_F(LoadCustomizableTest, LoadTableFactoryTest) {
}
}
TEST_F(LoadCustomizableTest, LoadFileSystemTest) {
ColumnFamilyOptions cf_opts;
std::shared_ptr<FileSystem> result;
ASSERT_NOK(FileSystem::CreateFromString(
config_options_, DummyFileSystem::kClassName(), &result));
ASSERT_OK(FileSystem::CreateFromString(config_options_,
FileSystem::kDefaultName(), &result));
ASSERT_NE(result, nullptr);
ASSERT_TRUE(result->IsInstanceOf(FileSystem::kDefaultName()));
if (RegisterTests("Test")) {
ASSERT_OK(FileSystem::CreateFromString(
config_options_, DummyFileSystem::kClassName(), &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), DummyFileSystem::kClassName());
ASSERT_FALSE(result->IsInstanceOf(FileSystem::kDefaultName()));
}
}
TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) {
std::shared_ptr<SecondaryCache> result;
ASSERT_NOK(SecondaryCache::CreateFromString(

5
port/win/env_win.h
View File

@ -110,7 +110,10 @@ class WinFileSystem : public FileSystem {
static const std::shared_ptr<WinFileSystem>& Default();
WinFileSystem(const std::shared_ptr<SystemClock>& clock);
~WinFileSystem() {}
const char* Name() const { return "WinFS"; }
static const char* kClassName() { return "WinFS"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const { return kDefaultName(); }
static size_t GetSectorSize(const std::string& fname);
size_t GetPageSize() const { return page_size_; }
size_t GetAllocationGranularity() const { return allocation_granularity_; }

3
test_util/testutil.h
View File

@ -555,6 +555,9 @@ class StringFS : public FileSystemWrapper {
: FileSystemWrapper(t) {}
~StringFS() override {}
static const char* kClassName() { return "StringFS"; }
const char* Name() const override { return kClassName(); }
const std::string& GetContent(const std::string& f) { return files_[f]; }
const IOStatus WriteToNewFile(const std::string& file_name,

266
utilities/env_timed.cc
View File

@ -2,6 +2,7 @@
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "utilities/env_timed.h"
#include "env/composite_env_wrapper.h"
#include "monitoring/perf_context_imp.h"
@ -12,154 +13,157 @@
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
namespace {
class TimedFileSystem : public FileSystemWrapper {
public:
explicit TimedFileSystem(const std::shared_ptr<FileSystem>& base)
: FileSystemWrapper(base) {}
TimedFileSystem::TimedFileSystem(const std::shared_ptr<FileSystem>& base)
: FileSystemWrapper(base) {}
IOStatus TimedFileSystem::NewSequentialFile(
const std::string& fname, const FileOptions& options,
std::unique_ptr<FSSequentialFile>* result, IODebugContext* dbg) {
PERF_TIMER_GUARD(env_new_sequential_file_nanos);
return FileSystemWrapper::NewSequentialFile(fname, options, result, dbg);
}
const char* Name() const override { return "TimedFS"; }
IOStatus NewSequentialFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSSequentialFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_sequential_file_nanos);
return FileSystemWrapper::NewSequentialFile(fname, options, result, dbg);
}
IOStatus TimedFileSystem::NewRandomAccessFile(
const std::string& fname, const FileOptions& options,
std::unique_ptr<FSRandomAccessFile>* result, IODebugContext* dbg) {
PERF_TIMER_GUARD(env_new_random_access_file_nanos);
return FileSystemWrapper::NewRandomAccessFile(fname, options, result, dbg);
}
IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSRandomAccessFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_random_access_file_nanos);
return FileSystemWrapper::NewRandomAccessFile(fname, options, result, dbg);
}
IOStatus TimedFileSystem::NewWritableFile(
const std::string& fname, const FileOptions& options,
std::unique_ptr<FSWritableFile>* result, IODebugContext* dbg) {
PERF_TIMER_GUARD(env_new_writable_file_nanos);
return FileSystemWrapper::NewWritableFile(fname, options, result, dbg);
}
IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_writable_file_nanos);
return FileSystemWrapper::NewWritableFile(fname, options, result, dbg);
}
IOStatus TimedFileSystem::ReuseWritableFile(
const std::string& fname, const std::string& old_fname,
const FileOptions& options, std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_reuse_writable_file_nanos);
return FileSystemWrapper::ReuseWritableFile(fname, old_fname, options, result,
dbg);
}
IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_reuse_writable_file_nanos);
return FileSystemWrapper::ReuseWritableFile(fname, old_fname, options,
result, dbg);
}
IOStatus TimedFileSystem::NewRandomRWFile(
const std::string& fname, const FileOptions& options,
std::unique_ptr<FSRandomRWFile>* result, IODebugContext* dbg) {
PERF_TIMER_GUARD(env_new_random_rw_file_nanos);
return FileSystemWrapper::NewRandomRWFile(fname, options, result, dbg);
}
IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_random_rw_file_nanos);
return FileSystemWrapper::NewRandomRWFile(fname, options, result, dbg);
}
IOStatus TimedFileSystem::NewDirectory(const std::string& name,
const IOOptions& options,
std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_new_directory_nanos);
return FileSystemWrapper::NewDirectory(name, options, result, dbg);
}
IOStatus NewDirectory(const std::string& name, const IOOptions& options,
std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_directory_nanos);
return FileSystemWrapper::NewDirectory(name, options, result, dbg);
}
IOStatus FileExists(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_file_exists_nanos);
return FileSystemWrapper::FileExists(fname, options, dbg);
}
IOStatus GetChildren(const std::string& dir, const IOOptions& options,
std::vector<std::string>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_children_nanos);
return FileSystemWrapper::GetChildren(dir, options, result, dbg);
}
IOStatus GetChildrenFileAttributes(const std::string& dir,
IOStatus TimedFileSystem::FileExists(const std::string& fname,
const IOOptions& options,
std::vector<FileAttributes>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_children_file_attributes_nanos);
return FileSystemWrapper::GetChildrenFileAttributes(dir, options, result,
dbg);
}
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_file_exists_nanos);
return FileSystemWrapper::FileExists(fname, options, dbg);
}
IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_delete_file_nanos);
return FileSystemWrapper::DeleteFile(fname, options, dbg);
}
IOStatus TimedFileSystem::GetChildren(const std::string& dir,
const IOOptions& options,
std::vector<std::string>* result,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_get_children_nanos);
return FileSystemWrapper::GetChildren(dir, options, result, dbg);
}
IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_create_dir_nanos);
return FileSystemWrapper::CreateDir(dirname, options, dbg);
}
IOStatus TimedFileSystem::GetChildrenFileAttributes(
const std::string& dir, const IOOptions& options,
std::vector<FileAttributes>* result, IODebugContext* dbg) {
PERF_TIMER_GUARD(env_get_children_file_attributes_nanos);
return FileSystemWrapper::GetChildrenFileAttributes(dir, options, result,
dbg);
}
IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_create_dir_if_missing_nanos);
return FileSystemWrapper::CreateDirIfMissing(dirname, options, dbg);
}
IOStatus TimedFileSystem::DeleteFile(const std::string& fname,
const IOOptions& options,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_delete_file_nanos);
return FileSystemWrapper::DeleteFile(fname, options, dbg);
}
IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_delete_dir_nanos);
return FileSystemWrapper::DeleteDir(dirname, options, dbg);
}
IOStatus TimedFileSystem::CreateDir(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_create_dir_nanos);
return FileSystemWrapper::CreateDir(dirname, options, dbg);
}
IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_file_size_nanos);
return FileSystemWrapper::GetFileSize(fname, options, file_size, dbg);
}
IOStatus TimedFileSystem::CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_create_dir_if_missing_nanos);
return FileSystemWrapper::CreateDirIfMissing(dirname, options, dbg);
}
IOStatus GetFileModificationTime(const std::string& fname,
IOStatus TimedFileSystem::DeleteDir(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_delete_dir_nanos);
return FileSystemWrapper::DeleteDir(dirname, options, dbg);
}
IOStatus TimedFileSystem::GetFileSize(const std::string& fname,
const IOOptions& options,
uint64_t* file_size,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_get_file_size_nanos);
return FileSystemWrapper::GetFileSize(fname, options, file_size, dbg);
}
IOStatus TimedFileSystem::GetFileModificationTime(const std::string& fname,
const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_get_file_modification_time_nanos);
return FileSystemWrapper::GetFileModificationTime(fname, options, file_mtime,
dbg);
}
IOStatus TimedFileSystem::RenameFile(const std::string& src,
const std::string& dst,
const IOOptions& options,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_rename_file_nanos);
return FileSystemWrapper::RenameFile(src, dst, options, dbg);
}
IOStatus TimedFileSystem::LinkFile(const std::string& src,
const std::string& dst,
const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_file_modification_time_nanos);
return FileSystemWrapper::GetFileModificationTime(fname, options,
file_mtime, dbg);
}
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_link_file_nanos);
return FileSystemWrapper::LinkFile(src, dst, options, dbg);
}
IOStatus RenameFile(const std::string& src, const std::string& dst,
const IOOptions& options, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_rename_file_nanos);
return FileSystemWrapper::RenameFile(src, dst, options, dbg);
}
IOStatus TimedFileSystem::LockFile(const std::string& fname,
const IOOptions& options, FileLock** lock,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_lock_file_nanos);
return FileSystemWrapper::LockFile(fname, options, lock, dbg);
}
IOStatus LinkFile(const std::string& src, const std::string& dst,
const IOOptions& options, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_link_file_nanos);
return FileSystemWrapper::LinkFile(src, dst, options, dbg);
}
IOStatus TimedFileSystem::UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_unlock_file_nanos);
return FileSystemWrapper::UnlockFile(lock, options, dbg);
}
IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_lock_file_nanos);
return FileSystemWrapper::LockFile(fname, options, lock, dbg);
}
IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_unlock_file_nanos);
return FileSystemWrapper::UnlockFile(lock, options, dbg);
}
IOStatus NewLogger(const std::string& fname, const IOOptions& options,
std::shared_ptr<Logger>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_logger_nanos);
return FileSystemWrapper::NewLogger(fname, options, result, dbg);
}
};
} // namespace
IOStatus TimedFileSystem::NewLogger(const std::string& fname,
const IOOptions& options,
std::shared_ptr<Logger>* result,
IODebugContext* dbg) {
PERF_TIMER_GUARD(env_new_logger_nanos);
return FileSystemWrapper::NewLogger(fname, options, result, dbg);
}
std::shared_ptr<FileSystem> NewTimedFileSystem(
const std::shared_ptr<FileSystem>& base) {

97
utilities/env_timed.h Normal file
View File

@ -0,0 +1,97 @@
// Copyright (c) 2019-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#pragma once
#include "rocksdb/file_system.h"
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
class TimedFileSystem : public FileSystemWrapper {
public:
explicit TimedFileSystem(const std::shared_ptr<FileSystem>& base);
static const char* kClassName() { return "TimedFS"; }
const char* Name() const override { return kClassName(); }
IOStatus NewSequentialFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSSequentialFile>* result,
IODebugContext* dbg) override;
IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSRandomAccessFile>* result,
IODebugContext* dbg) override;
IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* dbg) override;
IOStatus NewDirectory(const std::string& name, const IOOptions& options,
std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override;
IOStatus FileExists(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetChildren(const std::string& dir, const IOOptions& options,
std::vector<std::string>* result,
IODebugContext* dbg) override;
IOStatus GetChildrenFileAttributes(const std::string& dir,
const IOOptions& options,
std::vector<FileAttributes>* result,
IODebugContext* dbg) override;
IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override;
IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override;
IOStatus GetFileModificationTime(const std::string& fname,
const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override;
IOStatus RenameFile(const std::string& src, const std::string& dst,
const IOOptions& options, IODebugContext* dbg) override;
IOStatus LinkFile(const std::string& src, const std::string& dst,
const IOOptions& options, IODebugContext* dbg) override;
IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override;
IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus NewLogger(const std::string& fname, const IOOptions& options,
std::shared_ptr<Logger>* result,
IODebugContext* dbg) override;
};
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

3
utilities/fault_injection_fs.h
View File

@ -200,7 +200,8 @@ class FaultInjectionTestFS : public FileSystemWrapper {
fail_get_file_unique_id_(false) {}
virtual ~FaultInjectionTestFS() { error_.PermitUncheckedError(); }
const char* Name() const override { return "FaultInjectionTestFS"; }
static const char* kClassName() { return "FaultInjectionTestFS"; }
const char* Name() const override { return kClassName(); }
IOStatus NewDirectory(const std::string& name, const IOOptions& options,
std::unique_ptr<FSDirectory>* result,