2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2014-09-17 19:49:13 +00:00
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
2017-02-07 18:35:15 +00:00
|
|
|
#include <map>
|
2014-11-18 21:21:02 +00:00
|
|
|
#include <stdexcept>
|
2016-09-20 20:02:41 +00:00
|
|
|
#include <string>
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
2015-11-12 14:52:43 +00:00
|
|
|
#include <vector>
|
|
|
|
|
2017-04-06 02:02:00 +00:00
|
|
|
#include "options/cf_options.h"
|
|
|
|
#include "options/db_options.h"
|
2015-08-26 23:13:56 +00:00
|
|
|
#include "rocksdb/options.h"
|
2014-11-05 00:23:05 +00:00
|
|
|
#include "rocksdb/status.h"
|
2015-10-11 19:17:42 +00:00
|
|
|
#include "rocksdb/table.h"
|
2017-12-11 21:12:12 +00:00
|
|
|
#include "rocksdb/universal_compaction.h"
|
2014-09-17 19:49:13 +00:00
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
2016-09-23 23:34:04 +00:00
|
|
|
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
|
|
|
|
const MutableDBOptions& mutable_db_options);
|
|
|
|
|
2016-09-15 05:10:28 +00:00
|
|
|
ColumnFamilyOptions BuildColumnFamilyOptions(
|
|
|
|
const ColumnFamilyOptions& ioptions,
|
|
|
|
const MutableCFOptions& mutable_cf_options);
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
2014-11-05 00:23:05 +00:00
|
|
|
Status GetMutableOptionsFromStrings(
|
2014-09-17 19:49:13 +00:00
|
|
|
const MutableCFOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& options_map,
|
2018-04-11 01:48:49 +00:00
|
|
|
Logger* info_log, MutableCFOptions* new_options);
|
2014-09-17 19:49:13 +00:00
|
|
|
|
2016-10-14 19:25:39 +00:00
|
|
|
Status GetMutableDBOptionsFromStrings(
|
|
|
|
const MutableDBOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& options_map,
|
|
|
|
MutableDBOptions* new_options);
|
|
|
|
|
2015-10-11 19:17:42 +00:00
|
|
|
Status GetTableFactoryFromMap(
|
|
|
|
const std::string& factory_name,
|
|
|
|
const std::unordered_map<std::string, std::string>& opt_map,
|
2017-06-13 23:55:08 +00:00
|
|
|
std::shared_ptr<TableFactory>* table_factory,
|
|
|
|
bool ignore_unknown_options = false);
|
2015-10-11 19:17:42 +00:00
|
|
|
|
2015-08-18 20:30:18 +00:00
|
|
|
enum class OptionType {
|
|
|
|
kBoolean,
|
|
|
|
kInt,
|
2016-09-14 04:12:43 +00:00
|
|
|
kVectorInt,
|
2015-08-18 20:30:18 +00:00
|
|
|
kUInt,
|
|
|
|
kUInt32T,
|
|
|
|
kUInt64T,
|
|
|
|
kSizeT,
|
|
|
|
kString,
|
2015-08-26 23:13:56 +00:00
|
|
|
kDouble,
|
|
|
|
kCompactionStyle,
|
2017-03-02 18:08:49 +00:00
|
|
|
kCompactionPri,
|
2015-10-02 22:35:32 +00:00
|
|
|
kSliceTransform,
|
|
|
|
kCompressionType,
|
|
|
|
kVectorCompressionType,
|
|
|
|
kTableFactory,
|
|
|
|
kComparator,
|
|
|
|
kCompactionFilter,
|
|
|
|
kCompactionFilterFactory,
|
2017-10-19 22:19:20 +00:00
|
|
|
kCompactionOptionsFIFO,
|
2017-12-11 21:12:12 +00:00
|
|
|
kCompactionOptionsUniversal,
|
|
|
|
kCompactionStopStyle,
|
2015-10-02 22:35:32 +00:00
|
|
|
kMergeOperator,
|
|
|
|
kMemTableRepFactory,
|
2015-10-11 19:17:42 +00:00
|
|
|
kBlockBasedTableIndexType,
|
2018-07-27 22:35:41 +00:00
|
|
|
kBlockBasedTableDataBlockIndexType,
|
2015-10-11 19:17:42 +00:00
|
|
|
kFilterPolicy,
|
|
|
|
kFlushBlockPolicyFactory,
|
|
|
|
kChecksumType,
|
2015-10-28 17:46:01 +00:00
|
|
|
kEncodingType,
|
2016-03-02 18:34:14 +00:00
|
|
|
kWALRecoveryMode,
|
|
|
|
kAccessHint,
|
|
|
|
kInfoLogLevel,
|
2017-11-28 18:35:17 +00:00
|
|
|
kLRUCacheOptions,
|
2015-08-18 20:30:18 +00:00
|
|
|
kUnknown
|
|
|
|
};
|
|
|
|
|
2015-09-30 00:13:02 +00:00
|
|
|
enum class OptionVerificationType {
|
|
|
|
kNormal,
|
2017-10-04 16:44:18 +00:00
|
|
|
kByName, // The option is pointer typed so we can only verify
|
|
|
|
// based on it's name.
|
|
|
|
kByNameAllowNull, // Same as kByName, but it also allows the case
|
|
|
|
// where one of them is a nullptr.
|
|
|
|
kByNameAllowFromNull, // Same as kByName, but it also allows the case
|
|
|
|
// where the old option is nullptr.
|
|
|
|
kDeprecated // The option is no longer used in rocksdb. The RocksDB
|
|
|
|
// OptionsParser will still accept this option if it
|
|
|
|
// happen to exists in some Options file. However,
|
|
|
|
// the parser will not include it in serialization
|
|
|
|
// and verification processes.
|
2015-09-30 00:13:02 +00:00
|
|
|
};
|
|
|
|
|
2015-08-18 20:30:18 +00:00
|
|
|
// A struct for storing constant option information such as option name,
|
|
|
|
// option type, and offset.
|
|
|
|
struct OptionTypeInfo {
|
|
|
|
int offset;
|
|
|
|
OptionType type;
|
2015-09-30 00:13:02 +00:00
|
|
|
OptionVerificationType verification;
|
2016-09-14 04:12:43 +00:00
|
|
|
bool is_mutable;
|
|
|
|
int mutable_offset;
|
2015-08-18 20:30:18 +00:00
|
|
|
};
|
|
|
|
|
2015-10-02 22:35:32 +00:00
|
|
|
// A helper function that converts "opt_address" to a std::string
|
|
|
|
// based on the specified OptionType.
|
|
|
|
bool SerializeSingleOptionHelper(const char* opt_address,
|
|
|
|
const OptionType opt_type, std::string* value);
|
|
|
|
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
2015-11-12 14:52:43 +00:00
|
|
|
// In addition to its public version defined in rocksdb/convenience.h,
|
|
|
|
// this further takes an optional output vector "unsupported_options_names",
|
|
|
|
// which stores the name of all the unsupported options specified in "opts_map".
|
|
|
|
Status GetDBOptionsFromMapInternal(
|
|
|
|
const DBOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
DBOptions* new_options, bool input_strings_escaped,
|
2017-06-13 23:55:08 +00:00
|
|
|
std::vector<std::string>* unsupported_options_names = nullptr,
|
|
|
|
bool ignore_unknown_options = false);
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
2015-11-12 14:52:43 +00:00
|
|
|
|
|
|
|
// In addition to its public version defined in rocksdb/convenience.h,
|
|
|
|
// this further takes an optional output vector "unsupported_options_names",
|
|
|
|
// which stores the name of all the unsupported options specified in "opts_map".
|
|
|
|
Status GetColumnFamilyOptionsFromMapInternal(
|
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
ColumnFamilyOptions* new_options, bool input_strings_escaped,
|
2017-06-13 23:55:08 +00:00
|
|
|
std::vector<std::string>* unsupported_options_names = nullptr,
|
|
|
|
bool ignore_unknown_options = false);
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
2015-11-12 14:52:43 +00:00
|
|
|
|
2018-06-26 22:56:26 +00:00
|
|
|
bool ParseSliceTransform(
|
|
|
|
const std::string& value,
|
|
|
|
std::shared_ptr<const SliceTransform>* slice_transform);
|
|
|
|
|
2017-07-28 23:23:50 +00:00
|
|
|
extern Status StringToMap(
|
|
|
|
const std::string& opts_str,
|
|
|
|
std::unordered_map<std::string, std::string>* opts_map);
|
|
|
|
|
|
|
|
extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
|
|
|
|
const std::string& value);
|
2016-09-15 05:10:28 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
2017-11-18 01:02:13 +00:00
|
|
|
struct OptionsHelper {
|
|
|
|
static std::map<CompactionStyle, std::string> compaction_style_to_string;
|
|
|
|
static std::map<CompactionPri, std::string> compaction_pri_to_string;
|
|
|
|
static std::map<CompactionStopStyle, std::string>
|
|
|
|
compaction_stop_style_to_string;
|
|
|
|
static std::unordered_map<std::string, ChecksumType> checksum_type_string_map;
|
2018-01-23 22:36:54 +00:00
|
|
|
static std::unordered_map<std::string, CompressionType>
|
|
|
|
compression_type_string_map;
|
2017-11-18 01:02:13 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info;
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
fifo_compaction_options_type_info;
|
2017-12-11 21:12:12 +00:00
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
universal_compaction_options_type_info;
|
|
|
|
static std::unordered_map<std::string, CompactionStopStyle>
|
|
|
|
compaction_stop_style_string_map;
|
2017-11-18 01:02:13 +00:00
|
|
|
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info;
|
2017-11-28 18:35:17 +00:00
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
lru_cache_options_type_info;
|
2017-11-18 01:02:13 +00:00
|
|
|
static std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
|
|
|
|
block_base_table_index_type_string_map;
|
2018-07-27 22:35:41 +00:00
|
|
|
static std::unordered_map<std::string,
|
|
|
|
BlockBasedTableOptions::DataBlockIndexType>
|
|
|
|
block_base_table_data_block_index_type_string_map;
|
2017-11-18 01:02:13 +00:00
|
|
|
static std::unordered_map<std::string, EncodingType> encoding_type_string_map;
|
|
|
|
static std::unordered_map<std::string, CompactionStyle>
|
|
|
|
compaction_style_string_map;
|
|
|
|
static std::unordered_map<std::string, CompactionPri>
|
|
|
|
compaction_pri_string_map;
|
|
|
|
static std::unordered_map<std::string, WALRecoveryMode>
|
|
|
|
wal_recovery_mode_string_map;
|
|
|
|
static std::unordered_map<std::string, DBOptions::AccessHint>
|
|
|
|
access_hint_string_map;
|
|
|
|
static std::unordered_map<std::string, InfoLogLevel>
|
|
|
|
info_log_level_string_map;
|
|
|
|
static ColumnFamilyOptions dummy_cf_options;
|
|
|
|
static CompactionOptionsFIFO dummy_comp_options;
|
2017-11-28 18:35:17 +00:00
|
|
|
static LRUCacheOptions dummy_lru_cache_options;
|
2017-12-11 21:12:12 +00:00
|
|
|
static CompactionOptionsUniversal dummy_comp_options_universal;
|
2017-11-18 01:02:13 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
};
|
|
|
|
|
|
|
|
// Some aliasing
|
|
|
|
static auto& compaction_style_to_string =
|
|
|
|
OptionsHelper::compaction_style_to_string;
|
|
|
|
static auto& compaction_pri_to_string = OptionsHelper::compaction_pri_to_string;
|
|
|
|
static auto& compaction_stop_style_to_string =
|
|
|
|
OptionsHelper::compaction_stop_style_to_string;
|
|
|
|
static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
static auto& cf_options_type_info = OptionsHelper::cf_options_type_info;
|
|
|
|
static auto& fifo_compaction_options_type_info =
|
|
|
|
OptionsHelper::fifo_compaction_options_type_info;
|
2017-12-11 21:12:12 +00:00
|
|
|
static auto& universal_compaction_options_type_info =
|
|
|
|
OptionsHelper::universal_compaction_options_type_info;
|
|
|
|
static auto& compaction_stop_style_string_map =
|
|
|
|
OptionsHelper::compaction_stop_style_string_map;
|
2017-11-18 01:02:13 +00:00
|
|
|
static auto& db_options_type_info = OptionsHelper::db_options_type_info;
|
2017-11-28 18:35:17 +00:00
|
|
|
static auto& lru_cache_options_type_info =
|
|
|
|
OptionsHelper::lru_cache_options_type_info;
|
2017-11-18 01:02:13 +00:00
|
|
|
static auto& compression_type_string_map =
|
|
|
|
OptionsHelper::compression_type_string_map;
|
|
|
|
static auto& block_base_table_index_type_string_map =
|
|
|
|
OptionsHelper::block_base_table_index_type_string_map;
|
2018-07-27 22:35:41 +00:00
|
|
|
static auto& block_base_table_data_block_index_type_string_map =
|
|
|
|
OptionsHelper::block_base_table_data_block_index_type_string_map;
|
2017-11-18 01:02:13 +00:00
|
|
|
static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map;
|
|
|
|
static auto& compaction_style_string_map =
|
|
|
|
OptionsHelper::compaction_style_string_map;
|
|
|
|
static auto& compaction_pri_string_map =
|
|
|
|
OptionsHelper::compaction_pri_string_map;
|
|
|
|
static auto& wal_recovery_mode_string_map =
|
|
|
|
OptionsHelper::wal_recovery_mode_string_map;
|
|
|
|
static auto& access_hint_string_map = OptionsHelper::access_hint_string_map;
|
|
|
|
static auto& info_log_level_string_map =
|
|
|
|
OptionsHelper::info_log_level_string_map;
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
2014-09-17 19:49:13 +00:00
|
|
|
} // namespace rocksdb
|