mirror of https://github.com/facebook/rocksdb.git
Make simple BlockBasedTableOptions mutable (#10021)
Summary: In theory, there should be no danger in mutability, as table builders and readers work from copies of BlockBasedTableOptions. However, there is currently an unresolved read-write race that affecting SetOptions on BBTO fields. This should be generally acceptable for non-pointer options of 64 bits or less, but a fix is needed to make it mutability general here. See https://github.com/facebook/rocksdb/issues/10079 This change systematically sets all of those "simple" options (and future such options) as mutable. (Resurrecting this PR perhaps preferable to proposed https://github.com/facebook/rocksdb/issues/13063) Pull Request resolved: https://github.com/facebook/rocksdb/pull/10021 Test Plan: Some unit test updates. XXX comment added to stress test code Reviewed By: cbi42 Differential Revision: D64360967 Pulled By: pdillinger fbshipit-source-id: ff220fa778331852fe331b42b76ac4adfcd2d760
This commit is contained in:
Peter Dillinger
Facebook GitHub Bot
parent
8592517c89
commit
351d2fd2b6
|
|
@ -133,7 +133,9 @@ Status Cache::CreateFromString(const ConfigOptions& config_options,
|
|||
std::shared_ptr<Cache>* result) {
|
||||
Status status;
|
||||
std::shared_ptr<Cache> cache;
|
||||
if (value.find("://") == std::string::npos) {
|
||||
if (StartsWith(value, "null")) {
|
||||
cache = nullptr;
|
||||
} else if (value.find("://") == std::string::npos) {
|
||||
if (value.find('=') == std::string::npos) {
|
||||
cache = NewLRUCache(ParseSizeT(value));
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -563,7 +563,7 @@ TEST_P(DBBlockCacheTest1, WarmCacheWithBlocksDuringFlush) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) {
|
||||
TEST_F(DBBlockCacheTest, DynamicOptions) {
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||
|
|
@ -578,39 +578,70 @@ TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) {
|
|||
DestroyAndReopen(options);
|
||||
|
||||
std::string value(kValueSize, 'a');
|
||||
auto st = options.statistics;
|
||||
|
||||
for (size_t i = 1; i <= 5; i++) {
|
||||
ASSERT_OK(Put(std::to_string(i), value));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_EQ(1,
|
||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
size_t i = 1;
|
||||
ASSERT_OK(Put(std::to_string(i), value));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
|
||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
ASSERT_EQ(0,
|
||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
ASSERT_EQ(
|
||||
0, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
ASSERT_EQ(1,
|
||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
}
|
||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
|
||||
++i;
|
||||
ASSERT_OK(dbfull()->SetOptions(
|
||||
{{"block_based_table_factory", "{prepopulate_block_cache=kDisable;}"}}));
|
||||
|
||||
for (size_t i = 6; i <= kNumBlocks; i++) {
|
||||
ASSERT_OK(Put(std::to_string(i), value));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_EQ(0,
|
||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
ASSERT_OK(Put(std::to_string(i), value));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
|
||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
ASSERT_EQ(1,
|
||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
ASSERT_EQ(
|
||||
1, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
ASSERT_EQ(0,
|
||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
}
|
||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
|
||||
++i;
|
||||
ASSERT_OK(dbfull()->SetOptions({{"block_based_table_factory",
|
||||
"{prepopulate_block_cache=kFlushOnly;}"}}));
|
||||
|
||||
ASSERT_OK(Put(std::to_string(i), value));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
|
||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
|
||||
++i;
|
||||
// NOT YET SUPPORTED
|
||||
ASSERT_NOK(dbfull()->SetOptions(
|
||||
{{"block_based_table_factory", "{block_cache=null;}"}}));
|
||||
|
||||
// ASSERT_OK(Put(std::to_string(i), value));
|
||||
// ASSERT_OK(Flush());
|
||||
// ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
|
||||
// ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
// ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
// ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
// ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
|
||||
// ++i;
|
||||
ASSERT_NOK(dbfull()->SetOptions(
|
||||
{{"block_based_table_factory", "{block_cache=1M;}"}}));
|
||||
|
||||
// ASSERT_OK(Put(std::to_string(i), value));
|
||||
// ASSERT_OK(Flush());
|
||||
// ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
|
||||
// ASSERT_EQ(value, Get(std::to_string(i)));
|
||||
// ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||
// ASSERT_EQ(0, st->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||
// ASSERT_EQ(1, st->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -339,6 +339,16 @@ bool StressTest::BuildOptionsTable() {
|
|||
"2",
|
||||
}},
|
||||
{"max_sequential_skip_in_iterations", {"4", "8", "12"}},
|
||||
// XXX: BlockBasedTableOptions fields are mutable, but there is a
|
||||
// read-write race condition affecting them with SetOptions.
|
||||
// See https://github.com/facebook/rocksdb/issues/10079
|
||||
// {"block_based_table_factory", {"{block_size=2048}",
|
||||
// "{block_size=4096}"}},
|
||||
// Also TODO: a way here to modify many different BBTO fields
|
||||
// {"block_based_table_factory", {"{filter_policy=bloomfilter:2.34",
|
||||
// "{filter_policy=ribbonfilter:3.45",
|
||||
// "{filter_policy=ribbonfilter:4.56:-1",
|
||||
// "{filter_policy=ribbonfilter:6.67:3"}},
|
||||
};
|
||||
if (FLAGS_compaction_style == kCompactionStyleUniversal &&
|
||||
FLAGS_universal_max_read_amp > 0) {
|
||||
|
|
|
|||
|
|
@ -125,7 +125,26 @@ struct CacheUsageOptions {
|
|||
std::map<CacheEntryRole, CacheEntryRoleOptions> options_overrides;
|
||||
};
|
||||
|
||||
// For advanced user only
|
||||
// Configures how SST files using the block-based table format (standard)
|
||||
// are written and read.
|
||||
//
|
||||
// Except as specifically noted, all "simple" options here are "mutable" using
|
||||
// SetOptions(), with the caveat that only new table builders and new table
|
||||
// readers will pick up new options. This is nearly immediate effect for
|
||||
// SST building, but in the worst case, options affecting reads only take
|
||||
// effect for new files. (Unless the DB is closed and re-opened, table readers
|
||||
// can live as long as the SST file itself.)
|
||||
//
|
||||
// Examples (DB* db):
|
||||
// db->SetOptions({{"block_based_table_factory",
|
||||
// "{detect_filter_construct_corruption=true;}"}});
|
||||
// db->SetOptions({{"block_based_table_factory",
|
||||
// "{max_auto_readahead_size=0;block_size=8192;}"}}));
|
||||
// db->SetOptions({{"block_based_table_factory",
|
||||
// "{prepopulate_block_cache=kFlushOnly;}"}}));
|
||||
//
|
||||
// For now, "simple" options are only non-pointer options that are 64 bits or
|
||||
// less.
|
||||
struct BlockBasedTableOptions {
|
||||
static const char* kName() { return "BlockTableOptions"; }
|
||||
// @flush_block_policy_factory creates the instances of flush block policy.
|
||||
|
|
@ -256,13 +275,16 @@ struct BlockBasedTableOptions {
|
|||
// even though they have different checksum type.
|
||||
ChecksumType checksum = kXXH3;
|
||||
|
||||
// Disable block cache. If this is set to true,
|
||||
// then no block cache should be used, and the block_cache should
|
||||
// point to a nullptr object.
|
||||
// Disable block cache. If this is set to true, then no block cache
|
||||
// will be configured (block_cache reset to nullptr).
|
||||
//
|
||||
// This option cannot be used with SetOptions because it only affects
|
||||
// the value of `block_cache` during initialization.
|
||||
bool no_block_cache = false;
|
||||
|
||||
// If non-NULL use the specified cache for blocks.
|
||||
// If NULL, rocksdb will automatically create and use a 32MB internal cache.
|
||||
// If non-nullptr and no_block_cache == false, use the specified cache for
|
||||
// blocks. If nullptr and no_block_cache == false, a 32MB internal cache
|
||||
// will be created and used.
|
||||
std::shared_ptr<Cache> block_cache = nullptr;
|
||||
|
||||
// If non-NULL use the specified cache for pages read from device
|
||||
|
|
@ -468,10 +490,6 @@ struct BlockBasedTableOptions {
|
|||
// useful in detecting software bugs or CPU+memory malfunction.
|
||||
// Turning on this feature increases filter construction time by 30%.
|
||||
//
|
||||
// This parameter can be changed dynamically by
|
||||
// DB::SetOptions({{"block_based_table_factory",
|
||||
// "{detect_filter_construct_corruption=true;}"}});
|
||||
//
|
||||
// TODO: optimize this performance
|
||||
bool detect_filter_construct_corruption = false;
|
||||
|
||||
|
|
@ -602,13 +620,6 @@ struct BlockBasedTableOptions {
|
|||
// Found that 256 KB readahead size provides the best performance, based on
|
||||
// experiments, for auto readahead. Experiment data is in PR #3282.
|
||||
//
|
||||
// This parameter can be changed dynamically by
|
||||
// DB::SetOptions({{"block_based_table_factory",
|
||||
// "{max_auto_readahead_size=0;}"}}));
|
||||
//
|
||||
// Changing the value dynamically will only affect files opened after the
|
||||
// change.
|
||||
//
|
||||
// Default: 256 KB (256 * 1024).
|
||||
size_t max_auto_readahead_size = 256 * 1024;
|
||||
|
||||
|
|
@ -620,10 +631,6 @@ struct BlockBasedTableOptions {
|
|||
// further helps if the workload exhibits high temporal locality, where most
|
||||
// of the reads go to recently written data. This also helps in case of
|
||||
// Distributed FileSystem.
|
||||
//
|
||||
// This parameter can be changed dynamically by
|
||||
// DB::SetOptions({{"block_based_table_factory",
|
||||
// "{prepopulate_block_cache=kFlushOnly;}"}}));
|
||||
enum class PrepopulateBlockCache : char {
|
||||
// Disable prepopulate block cache.
|
||||
kDisable,
|
||||
|
|
@ -653,13 +660,6 @@ struct BlockBasedTableOptions {
|
|||
// Value should be provided along with KB i.e. 8 * 1024 as it will prefetch
|
||||
// the blocks.
|
||||
//
|
||||
// This parameter can be changed dynamically by
|
||||
// DB::SetOptions({{"block_based_table_factory",
|
||||
// "{initial_auto_readahead_size=0;}"}}));
|
||||
//
|
||||
// Changing the value dynamically will only affect files opened after the
|
||||
// change.
|
||||
//
|
||||
// Default: 8 KB (8 * 1024).
|
||||
size_t initial_auto_readahead_size = 8 * 1024;
|
||||
|
||||
|
|
|
|||
|
|
@ -241,42 +241,13 @@ using ValidateFunc = std::function<Status(
|
|||
// option type, and offset.
|
||||
class OptionTypeInfo {
|
||||
public:
|
||||
// A simple "normal", non-mutable Type "type" at offset
|
||||
OptionTypeInfo(int offset, OptionType type)
|
||||
: offset_(offset),
|
||||
parse_func_(nullptr),
|
||||
serialize_func_(nullptr),
|
||||
equals_func_(nullptr),
|
||||
type_(type),
|
||||
verification_(OptionVerificationType::kNormal),
|
||||
flags_(OptionTypeFlags::kNone) {}
|
||||
|
||||
OptionTypeInfo(int offset, OptionType type,
|
||||
OptionVerificationType verification, OptionTypeFlags flags)
|
||||
: offset_(offset),
|
||||
parse_func_(nullptr),
|
||||
serialize_func_(nullptr),
|
||||
equals_func_(nullptr),
|
||||
type_(type),
|
||||
verification_(verification),
|
||||
flags_(flags) {}
|
||||
|
||||
OptionTypeInfo(int offset, OptionType type,
|
||||
OptionVerificationType verification, OptionTypeFlags flags,
|
||||
const ParseFunc& parse_func)
|
||||
: offset_(offset),
|
||||
parse_func_(parse_func),
|
||||
serialize_func_(nullptr),
|
||||
equals_func_(nullptr),
|
||||
type_(type),
|
||||
verification_(verification),
|
||||
flags_(flags) {}
|
||||
|
||||
OptionTypeInfo(int offset, OptionType type,
|
||||
OptionVerificationType verification, OptionTypeFlags flags,
|
||||
const ParseFunc& parse_func,
|
||||
const SerializeFunc& serialize_func,
|
||||
const EqualsFunc& equals_func)
|
||||
OptionTypeInfo(
|
||||
int offset, OptionType type,
|
||||
OptionVerificationType verification = OptionVerificationType::kNormal,
|
||||
OptionTypeFlags flags = OptionTypeFlags::kNone,
|
||||
const ParseFunc& parse_func = {},
|
||||
const SerializeFunc& serialize_func = {},
|
||||
const EqualsFunc& equals_func = {})
|
||||
: offset_(offset),
|
||||
parse_func_(parse_func),
|
||||
serialize_func_(serialize_func),
|
||||
|
|
@ -476,18 +447,17 @@ class OptionTypeInfo {
|
|||
//
|
||||
// @param offset The offset for the Customizable from the base pointer
|
||||
// @param ovt How to verify this option
|
||||
// @param flags, Extra flags specifying the behavior of this option
|
||||
// @param _sfunc Optional function for serializing this option
|
||||
// @param _efunc Optional function for comparing this option
|
||||
// @param flags Extra flags specifying the behavior of this option
|
||||
// @param serialize_func Optional function for serializing this option
|
||||
// @param equals_func Optional function for comparing this option
|
||||
template <typename T>
|
||||
static OptionTypeInfo AsCustomSharedPtr(int offset,
|
||||
OptionVerificationType ovt,
|
||||
OptionTypeFlags flags) {
|
||||
OptionTypeInfo info(offset, OptionType::kCustomizable, ovt,
|
||||
flags | OptionTypeFlags::kShared);
|
||||
return info.SetParseFunc([](const ConfigOptions& opts,
|
||||
const std::string& name,
|
||||
const std::string& value, void* addr) {
|
||||
static OptionTypeInfo AsCustomSharedPtr(
|
||||
int offset, OptionVerificationType ovt,
|
||||
OptionTypeFlags flags = OptionTypeFlags::kNone,
|
||||
const SerializeFunc& serialize_func = {},
|
||||
const EqualsFunc& equals_func = {}) {
|
||||
auto parse_func = [](const ConfigOptions& opts, const std::string& name,
|
||||
const std::string& value, void* addr) {
|
||||
auto* shared = static_cast<std::shared_ptr<T>*>(addr);
|
||||
if (name == kIdPropName() && value.empty()) {
|
||||
shared->reset();
|
||||
|
|
@ -495,19 +465,10 @@ class OptionTypeInfo {
|
|||
} else {
|
||||
return T::CreateFromString(opts, value, shared);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static OptionTypeInfo AsCustomSharedPtr(int offset,
|
||||
OptionVerificationType ovt,
|
||||
OptionTypeFlags flags,
|
||||
const SerializeFunc& serialize_func,
|
||||
const EqualsFunc& equals_func) {
|
||||
OptionTypeInfo info(AsCustomSharedPtr<T>(offset, ovt, flags));
|
||||
info.SetSerializeFunc(serialize_func);
|
||||
info.SetEqualsFunc(equals_func);
|
||||
return info;
|
||||
};
|
||||
return OptionTypeInfo(offset, OptionType::kCustomizable, ovt,
|
||||
flags | OptionTypeFlags::kShared, parse_func,
|
||||
serialize_func, equals_func);
|
||||
}
|
||||
|
||||
// Create a new std::unique_ptr<Customizable> OptionTypeInfo
|
||||
|
|
@ -612,6 +573,9 @@ class OptionTypeInfo {
|
|||
return *this;
|
||||
}
|
||||
|
||||
OptionTypeFlags GetFlags() const { return flags_; }
|
||||
void SetFlags(OptionTypeFlags flags) { flags_ = flags; }
|
||||
|
||||
bool IsEnabled(OptionTypeFlags otf) const { return (flags_ & otf) == otf; }
|
||||
|
||||
bool IsEditable(const ConfigOptions& opts) const {
|
||||
|
|
@ -714,6 +678,8 @@ class OptionTypeInfo {
|
|||
|
||||
bool IsCustomizable() const { return (type_ == OptionType::kCustomizable); }
|
||||
|
||||
OptionType GetType() const { return type_; }
|
||||
|
||||
inline const void* GetOffset(const void* base) const {
|
||||
return static_cast<const char*>(base) + offset_;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1662,28 +1662,28 @@ TEST_F(OptionsTest, MutableTableOptions) {
|
|||
bbtf.reset(NewBlockBasedTableFactory());
|
||||
auto bbto = bbtf->GetOptions<BlockBasedTableOptions>();
|
||||
ASSERT_NE(bbto, nullptr);
|
||||
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_align", "true"));
|
||||
ASSERT_OK(bbtf->ConfigureOption(config_options, "no_block_cache", "true"));
|
||||
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024"));
|
||||
ASSERT_EQ(bbto->block_align, true);
|
||||
ASSERT_EQ(bbto->no_block_cache, true);
|
||||
ASSERT_EQ(bbto->block_size, 1024);
|
||||
ASSERT_OK(bbtf->PrepareOptions(config_options));
|
||||
config_options.mutable_options_only = true;
|
||||
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024"));
|
||||
ASSERT_EQ(bbto->block_align, true);
|
||||
ASSERT_NOK(bbtf->ConfigureOption(config_options, "block_align", "false"));
|
||||
ASSERT_EQ(bbto->no_block_cache, true);
|
||||
ASSERT_NOK(bbtf->ConfigureOption(config_options, "no_block_cache", "false"));
|
||||
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "2048"));
|
||||
ASSERT_EQ(bbto->block_align, true);
|
||||
ASSERT_EQ(bbto->no_block_cache, true);
|
||||
ASSERT_EQ(bbto->block_size, 2048);
|
||||
|
||||
ColumnFamilyOptions cf_opts;
|
||||
cf_opts.table_factory = bbtf;
|
||||
ASSERT_NOK(GetColumnFamilyOptionsFromString(
|
||||
config_options, cf_opts, "block_based_table_factory.block_align=false",
|
||||
config_options, cf_opts, "block_based_table_factory.no_block_cache=false",
|
||||
&cf_opts));
|
||||
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
||||
config_options, cf_opts, "block_based_table_factory.block_size=8192",
|
||||
&cf_opts));
|
||||
ASSERT_EQ(bbto->block_align, true);
|
||||
ASSERT_EQ(bbto->no_block_cache, true);
|
||||
ASSERT_EQ(bbto->block_size, 8192);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -224,10 +224,20 @@ static std::unordered_map<std::string,
|
|||
{"kFlushOnly",
|
||||
BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly}};
|
||||
|
||||
static std::unordered_map<std::string, OptionTypeInfo>
|
||||
block_based_table_type_info = {
|
||||
/* currently not supported
|
||||
std::shared_ptr<Cache> block_cache = nullptr;
|
||||
static struct BlockBasedTableTypeInfo {
|
||||
std::unordered_map<std::string, OptionTypeInfo> info;
|
||||
|
||||
BlockBasedTableTypeInfo() {
|
||||
info = {
|
||||
// NOTE: Below the list, most of these options are marked as mutable.
|
||||
// In theory, there should be no danger in mutability, as table
|
||||
// builders and readers work from copies of BlockBasedTableOptions.
|
||||
// However, there is currently an unresolved read-write race that
|
||||
// affecting SetOptions on BBTO fields. This should be generally
|
||||
// acceptable for non-pointer options of 64 bits or less, but a fix
|
||||
// is needed to make it mutability general here. See
|
||||
// https://github.com/facebook/rocksdb/issues/10079
|
||||
/* currently not supported:
|
||||
CacheUsageOptions cache_usage_options;
|
||||
*/
|
||||
{"flush_block_policy_factory",
|
||||
|
|
@ -238,24 +248,20 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||
{"cache_index_and_filter_blocks",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
cache_index_and_filter_blocks),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"cache_index_and_filter_blocks_with_high_priority",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
cache_index_and_filter_blocks_with_high_priority),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"pin_l0_filter_and_index_blocks_in_cache",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
pin_l0_filter_and_index_blocks_in_cache),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"index_type", OptionTypeInfo::Enum<BlockBasedTableOptions::IndexType>(
|
||||
offsetof(struct BlockBasedTableOptions, index_type),
|
||||
&block_base_table_index_type_string_map)},
|
||||
{"hash_index_allow_collision",
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
||||
OptionTypeFlags::kNone}},
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated}},
|
||||
{"data_block_index_type",
|
||||
OptionTypeInfo::Enum<BlockBasedTableOptions::DataBlockIndexType>(
|
||||
offsetof(struct BlockBasedTableOptions, data_block_index_type),
|
||||
|
|
@ -267,86 +273,65 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||
{"data_block_hash_table_util_ratio",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
data_block_hash_table_util_ratio),
|
||||
OptionType::kDouble, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kDouble, OptionVerificationType::kNormal}},
|
||||
{"checksum",
|
||||
{offsetof(struct BlockBasedTableOptions, checksum),
|
||||
OptionType::kChecksumType, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kChecksumType, OptionVerificationType::kNormal}},
|
||||
{"no_block_cache",
|
||||
{offsetof(struct BlockBasedTableOptions, no_block_cache),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"block_size",
|
||||
{offsetof(struct BlockBasedTableOptions, block_size),
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kMutable}},
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal}},
|
||||
{"block_size_deviation",
|
||||
{offsetof(struct BlockBasedTableOptions, block_size_deviation),
|
||||
OptionType::kInt, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kInt, OptionVerificationType::kNormal}},
|
||||
{"block_restart_interval",
|
||||
{offsetof(struct BlockBasedTableOptions, block_restart_interval),
|
||||
OptionType::kInt, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kMutable}},
|
||||
OptionType::kInt, OptionVerificationType::kNormal}},
|
||||
{"index_block_restart_interval",
|
||||
{offsetof(struct BlockBasedTableOptions, index_block_restart_interval),
|
||||
OptionType::kInt, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kInt, OptionVerificationType::kNormal}},
|
||||
{"index_per_partition",
|
||||
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
|
||||
OptionTypeFlags::kNone}},
|
||||
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated}},
|
||||
{"metadata_block_size",
|
||||
{offsetof(struct BlockBasedTableOptions, metadata_block_size),
|
||||
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kUInt64T, OptionVerificationType::kNormal}},
|
||||
{"partition_filters",
|
||||
{offsetof(struct BlockBasedTableOptions, partition_filters),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"decouple_partitioned_filters",
|
||||
{offsetof(struct BlockBasedTableOptions, decouple_partitioned_filters),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"optimize_filters_for_memory",
|
||||
{offsetof(struct BlockBasedTableOptions, optimize_filters_for_memory),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"use_delta_encoding",
|
||||
{offsetof(struct BlockBasedTableOptions, use_delta_encoding),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"filter_policy",
|
||||
OptionTypeInfo::AsCustomSharedPtr<const FilterPolicy>(
|
||||
offsetof(struct BlockBasedTableOptions, filter_policy),
|
||||
OptionVerificationType::kByNameAllowFromNull,
|
||||
OptionTypeFlags::kNone)},
|
||||
OptionVerificationType::kByNameAllowFromNull)},
|
||||
{"whole_key_filtering",
|
||||
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"detect_filter_construct_corruption",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
detect_filter_construct_corruption),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kMutable}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"reserve_table_builder_memory",
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
||||
OptionTypeFlags::kNone}},
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated}},
|
||||
{"reserve_table_reader_memory",
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
||||
OptionTypeFlags::kNone}},
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated}},
|
||||
{"skip_table_builder_flush",
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
||||
OptionTypeFlags::kNone}},
|
||||
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated}},
|
||||
{"format_version",
|
||||
{offsetof(struct BlockBasedTableOptions, format_version),
|
||||
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kUInt32T, OptionVerificationType::kNormal}},
|
||||
{"verify_compression",
|
||||
{offsetof(struct BlockBasedTableOptions, verify_compression),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"read_amp_bytes_per_bit",
|
||||
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
|
||||
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
||||
|
|
@ -369,17 +354,14 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||
}}},
|
||||
{"enable_index_compression",
|
||||
{offsetof(struct BlockBasedTableOptions, enable_index_compression),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"block_align",
|
||||
{offsetof(struct BlockBasedTableOptions, block_align),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"pin_top_level_index_and_filter",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
pin_top_level_index_and_filter),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kNone}},
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{kOptNameMetadataCacheOpts,
|
||||
OptionTypeInfo::Struct(
|
||||
kOptNameMetadataCacheOpts, &metadata_cache_options_type_info,
|
||||
|
|
@ -396,28 +378,52 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||
return Cache::CreateFromString(opts, value, cache);
|
||||
}}},
|
||||
{"block_cache_compressed",
|
||||
{0, OptionType::kUnknown, OptionVerificationType::kDeprecated,
|
||||
OptionTypeFlags::kNone}},
|
||||
{0, OptionType::kUnknown, OptionVerificationType::kDeprecated}},
|
||||
{"max_auto_readahead_size",
|
||||
{offsetof(struct BlockBasedTableOptions, max_auto_readahead_size),
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kMutable}},
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal}},
|
||||
{"prepopulate_block_cache",
|
||||
OptionTypeInfo::Enum<BlockBasedTableOptions::PrepopulateBlockCache>(
|
||||
offsetof(struct BlockBasedTableOptions, prepopulate_block_cache),
|
||||
&block_base_table_prepopulate_block_cache_string_map,
|
||||
OptionTypeFlags::kMutable)},
|
||||
&block_base_table_prepopulate_block_cache_string_map)},
|
||||
{"initial_auto_readahead_size",
|
||||
{offsetof(struct BlockBasedTableOptions, initial_auto_readahead_size),
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kMutable}},
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal}},
|
||||
{"num_file_reads_for_auto_readahead",
|
||||
{offsetof(struct BlockBasedTableOptions,
|
||||
num_file_reads_for_auto_readahead),
|
||||
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
||||
OptionTypeFlags::kMutable}},
|
||||
OptionType::kUInt64T, OptionVerificationType::kNormal}},
|
||||
};
|
||||
|
||||
};
|
||||
for (auto& i : info) {
|
||||
switch (i.second.GetType()) {
|
||||
case OptionType::kString:
|
||||
case OptionType::kEnv:
|
||||
case OptionType::kStruct:
|
||||
case OptionType::kVector:
|
||||
case OptionType::kConfigurable:
|
||||
case OptionType::kCustomizable:
|
||||
case OptionType::kEncodedString:
|
||||
case OptionType::kArray:
|
||||
case OptionType::kUnknown:
|
||||
// Not mutable, at least until race condition is resolved
|
||||
break;
|
||||
default:
|
||||
if (i.first == "no_block_cache") {
|
||||
// Special case of not being mutable; only makes sense at
|
||||
// initialization time, and for non-expert users
|
||||
assert((i.second.GetFlags() & OptionTypeFlags::kMutable) ==
|
||||
OptionTypeFlags::kNone);
|
||||
} else {
|
||||
// Values up to 64 bits should be generally safe to mutate despite
|
||||
// the read-write race.
|
||||
i.second.SetFlags(i.second.GetFlags() | OptionTypeFlags::kMutable);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} block_based_table_type_info;
|
||||
|
||||
// TODO(myabandeh): We should return an error instead of silently changing the
|
||||
// options
|
||||
|
|
@ -425,7 +431,7 @@ BlockBasedTableFactory::BlockBasedTableFactory(
|
|||
const BlockBasedTableOptions& _table_options)
|
||||
: table_options_(_table_options) {
|
||||
InitializeOptions();
|
||||
RegisterOptions(&table_options_, &block_based_table_type_info);
|
||||
RegisterOptions(&table_options_, &block_based_table_type_info.info);
|
||||
|
||||
const auto table_reader_charged =
|
||||
table_options_.cache_usage_options.options_overrides
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
* All "simple" options in `BlockBasedTableOptions` are now mutable with `DB::SetOptions()`. For now, "simple" only includes non-pointer options that are 64 bits or less.
|
||||
Loading…
Reference in New Issue