2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2017-04-06 02:02:00 +00:00
|
|
|
#include "options/options_helper.h"
|
2014-09-17 19:46:32 +00:00
|
|
|
|
2023-09-15 22:46:10 +00:00
|
|
|
#include <atomic>
|
2014-09-17 19:46:32 +00:00
|
|
|
#include <cassert>
|
2014-10-10 17:00:12 +00:00
|
|
|
#include <cctype>
|
2015-04-24 02:17:57 +00:00
|
|
|
#include <cstdlib>
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
#include <set>
|
2014-10-01 23:19:16 +00:00
|
|
|
#include <unordered_set>
|
2015-10-02 22:35:32 +00:00
|
|
|
#include <vector>
|
2019-05-30 21:47:29 +00:00
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "options/cf_options.h"
|
|
|
|
#include "options/db_options.h"
|
2014-12-22 21:18:57 +00:00
|
|
|
#include "rocksdb/cache.h"
|
2015-10-02 22:35:32 +00:00
|
|
|
#include "rocksdb/compaction_filter.h"
|
2015-07-15 21:51:51 +00:00
|
|
|
#include "rocksdb/convenience.h"
|
2014-12-22 21:18:57 +00:00
|
|
|
#include "rocksdb/filter_policy.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "rocksdb/flush_block_policy.h"
|
2015-10-02 22:35:32 +00:00
|
|
|
#include "rocksdb/memtablerep.h"
|
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-09-17 19:46:32 +00:00
|
|
|
#include "rocksdb/options.h"
|
2015-03-06 22:21:15 +00:00
|
|
|
#include "rocksdb/rate_limiter.h"
|
2015-01-15 23:33:12 +00:00
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-12-22 21:18:57 +00:00
|
|
|
#include "rocksdb/table.h"
|
2019-03-26 21:15:26 +00:00
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
2020-09-14 23:59:00 +00:00
|
|
|
#include "rocksdb/utilities/options_type.h"
|
2015-08-18 20:30:18 +00:00
|
|
|
#include "util/string_util.h"
|
2014-09-17 19:46:32 +00:00
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2021-05-11 13:45:49 +00:00
|
|
|
ConfigOptions::ConfigOptions()
|
|
|
|
: registry(ObjectRegistry::NewInstance())
|
|
|
|
{
|
|
|
|
env = Env::Default();
|
|
|
|
}
|
|
|
|
|
|
|
|
ConfigOptions::ConfigOptions(const DBOptions& db_opts) : env(db_opts.env) {
|
|
|
|
registry = ObjectRegistry::NewInstance();
|
|
|
|
}
|
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
Status ValidateOptions(const DBOptions& db_opts,
|
|
|
|
const ColumnFamilyOptions& cf_opts) {
|
|
|
|
Status s;
|
|
|
|
auto db_cfg = DBOptionsAsConfigurable(db_opts);
|
|
|
|
auto cf_cfg = CFOptionsAsConfigurable(cf_opts);
|
|
|
|
s = db_cfg->ValidateOptions(db_opts, cf_opts);
|
2023-12-01 19:10:30 +00:00
|
|
|
if (s.ok()) {
|
|
|
|
s = cf_cfg->ValidateOptions(db_opts, cf_opts);
|
|
|
|
}
|
2020-09-14 23:59:00 +00:00
|
|
|
return s;
|
|
|
|
}
|
2014-09-17 19:46:32 +00:00
|
|
|
|
2016-09-23 23:34:04 +00:00
|
|
|
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
|
|
|
|
const MutableDBOptions& mutable_db_options) {
|
|
|
|
DBOptions options;
|
2024-09-26 21:36:29 +00:00
|
|
|
BuildDBOptions(immutable_db_options, mutable_db_options, options);
|
|
|
|
return options;
|
|
|
|
}
|
2016-09-23 23:34:04 +00:00
|
|
|
|
2024-09-26 21:36:29 +00:00
|
|
|
void BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
|
|
|
|
const MutableDBOptions& mutable_db_options,
|
|
|
|
DBOptions& options) {
|
2016-09-23 23:34:04 +00:00
|
|
|
options.create_if_missing = immutable_db_options.create_if_missing;
|
|
|
|
options.create_missing_column_families =
|
|
|
|
immutable_db_options.create_missing_column_families;
|
|
|
|
options.error_if_exists = immutable_db_options.error_if_exists;
|
|
|
|
options.paranoid_checks = immutable_db_options.paranoid_checks;
|
2021-05-20 23:06:12 +00:00
|
|
|
options.flush_verify_memtable_count =
|
|
|
|
immutable_db_options.flush_verify_memtable_count;
|
2023-07-28 16:47:31 +00:00
|
|
|
options.compaction_verify_record_count =
|
|
|
|
immutable_db_options.compaction_verify_record_count;
|
2020-10-09 23:40:25 +00:00
|
|
|
options.track_and_verify_wals_in_manifest =
|
|
|
|
immutable_db_options.track_and_verify_wals_in_manifest;
|
2022-05-19 18:04:21 +00:00
|
|
|
options.verify_sst_unique_id_in_manifest =
|
|
|
|
immutable_db_options.verify_sst_unique_id_in_manifest;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.env = immutable_db_options.env;
|
|
|
|
options.rate_limiter = immutable_db_options.rate_limiter;
|
|
|
|
options.sst_file_manager = immutable_db_options.sst_file_manager;
|
|
|
|
options.info_log = immutable_db_options.info_log;
|
|
|
|
options.info_log_level = immutable_db_options.info_log_level;
|
2017-05-04 03:46:17 +00:00
|
|
|
options.max_open_files = mutable_db_options.max_open_files;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.max_file_opening_threads =
|
|
|
|
immutable_db_options.max_file_opening_threads;
|
2016-11-15 06:45:16 +00:00
|
|
|
options.max_total_wal_size = mutable_db_options.max_total_wal_size;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.statistics = immutable_db_options.statistics;
|
|
|
|
options.use_fsync = immutable_db_options.use_fsync;
|
|
|
|
options.db_paths = immutable_db_options.db_paths;
|
|
|
|
options.db_log_dir = immutable_db_options.db_log_dir;
|
|
|
|
options.wal_dir = immutable_db_options.wal_dir;
|
|
|
|
options.delete_obsolete_files_period_micros =
|
2016-12-05 22:09:35 +00:00
|
|
|
mutable_db_options.delete_obsolete_files_period_micros;
|
2017-05-24 18:25:38 +00:00
|
|
|
options.max_background_jobs = mutable_db_options.max_background_jobs;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.max_background_compactions =
|
2016-10-14 19:25:39 +00:00
|
|
|
mutable_db_options.max_background_compactions;
|
2020-07-23 01:31:25 +00:00
|
|
|
options.max_subcompactions = mutable_db_options.max_subcompactions;
|
2020-04-20 23:17:25 +00:00
|
|
|
options.max_background_flushes = mutable_db_options.max_background_flushes;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.max_log_file_size = immutable_db_options.max_log_file_size;
|
|
|
|
options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll;
|
|
|
|
options.keep_log_file_num = immutable_db_options.keep_log_file_num;
|
|
|
|
options.recycle_log_file_num = immutable_db_options.recycle_log_file_num;
|
|
|
|
options.max_manifest_file_size = immutable_db_options.max_manifest_file_size;
|
|
|
|
options.table_cache_numshardbits =
|
|
|
|
immutable_db_options.table_cache_numshardbits;
|
2021-04-23 03:42:50 +00:00
|
|
|
options.WAL_ttl_seconds = immutable_db_options.WAL_ttl_seconds;
|
|
|
|
options.WAL_size_limit_MB = immutable_db_options.WAL_size_limit_MB;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.manifest_preallocation_size =
|
|
|
|
immutable_db_options.manifest_preallocation_size;
|
|
|
|
options.allow_mmap_reads = immutable_db_options.allow_mmap_reads;
|
|
|
|
options.allow_mmap_writes = immutable_db_options.allow_mmap_writes;
|
2016-10-28 17:36:05 +00:00
|
|
|
options.use_direct_reads = immutable_db_options.use_direct_reads;
|
2017-04-13 20:07:33 +00:00
|
|
|
options.use_direct_io_for_flush_and_compaction =
|
|
|
|
immutable_db_options.use_direct_io_for_flush_and_compaction;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.allow_fallocate = immutable_db_options.allow_fallocate;
|
|
|
|
options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec;
|
2017-03-21 05:50:56 +00:00
|
|
|
options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec;
|
2019-02-20 23:46:59 +00:00
|
|
|
options.stats_persist_period_sec =
|
|
|
|
mutable_db_options.stats_persist_period_sec;
|
2019-06-17 22:17:43 +00:00
|
|
|
options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk;
|
2019-02-20 23:46:59 +00:00
|
|
|
options.stats_history_buffer_size =
|
|
|
|
mutable_db_options.stats_history_buffer_size;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.advise_random_on_open = immutable_db_options.advise_random_on_open;
|
|
|
|
options.db_write_buffer_size = immutable_db_options.db_write_buffer_size;
|
|
|
|
options.write_buffer_manager = immutable_db_options.write_buffer_manager;
|
|
|
|
options.compaction_readahead_size =
|
2017-11-17 01:46:43 +00:00
|
|
|
mutable_db_options.compaction_readahead_size;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.random_access_max_buffer_size =
|
|
|
|
immutable_db_options.random_access_max_buffer_size;
|
|
|
|
options.writable_file_max_buffer_size =
|
2017-10-31 20:49:25 +00:00
|
|
|
mutable_db_options.writable_file_max_buffer_size;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
|
2024-09-26 21:36:29 +00:00
|
|
|
options.bytes_per_sync = mutable_db_options.bytes_per_sync;
|
|
|
|
options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync;
|
|
|
|
options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.listeners = immutable_db_options.listeners;
|
|
|
|
options.enable_thread_tracking = immutable_db_options.enable_thread_tracking;
|
2016-11-12 23:43:33 +00:00
|
|
|
options.delayed_write_rate = mutable_db_options.delayed_write_rate;
|
2018-03-13 18:50:16 +00:00
|
|
|
options.enable_pipelined_write = immutable_db_options.enable_pipelined_write;
|
2019-05-14 00:43:47 +00:00
|
|
|
options.unordered_write = immutable_db_options.unordered_write;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.allow_concurrent_memtable_write =
|
|
|
|
immutable_db_options.allow_concurrent_memtable_write;
|
|
|
|
options.enable_write_thread_adaptive_yield =
|
|
|
|
immutable_db_options.enable_write_thread_adaptive_yield;
|
2019-09-12 01:26:22 +00:00
|
|
|
options.max_write_batch_group_size_bytes =
|
|
|
|
immutable_db_options.max_write_batch_group_size_bytes;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.write_thread_max_yield_usec =
|
|
|
|
immutable_db_options.write_thread_max_yield_usec;
|
|
|
|
options.write_thread_slow_yield_usec =
|
|
|
|
immutable_db_options.write_thread_slow_yield_usec;
|
|
|
|
options.skip_stats_update_on_db_open =
|
|
|
|
immutable_db_options.skip_stats_update_on_db_open;
|
Add an option to prevent DB::Open() from querying sizes of all sst files (#6353)
Summary:
When paranoid_checks is on, DBImpl::CheckConsistency() iterates over all sst files and calls Env::GetFileSize() for each of them. As far as I could understand, this is pretty arbitrary and doesn't affect correctness - if filesystem doesn't corrupt fsynced files, the file sizes will always match; if it does, it may as well corrupt contents as well as sizes, and rocksdb doesn't check contents on open.
If there are thousands of sst files, getting all their sizes takes a while. If, on top of that, Env is overridden to use some remote storage instead of local filesystem, it can be *really* slow and overload the remote storage service. This PR adds an option to not do GetFileSize(); instead it does GetChildren() for parent directory to check that all the expected sst files are at least present, but doesn't check their sizes.
We can't just disable paranoid_checks instead because paranoid_checks do a few other important things: make the DB read-only on write errors, print error messages on read errors, etc.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6353
Test Plan: ran the added sanity check unit test. Will try it out in a LogDevice test cluster where the GetFileSize() calls are causing a lot of trouble.
Differential Revision: D19656425
Pulled By: al13n321
fbshipit-source-id: c2c421b367633033760d1f56747bad206d1fbf82
2020-02-04 09:24:29 +00:00
|
|
|
options.skip_checking_sst_file_sizes_on_db_open =
|
|
|
|
immutable_db_options.skip_checking_sst_file_sizes_on_db_open;
|
2016-09-23 23:34:04 +00:00
|
|
|
options.wal_recovery_mode = immutable_db_options.wal_recovery_mode;
|
|
|
|
options.allow_2pc = immutable_db_options.allow_2pc;
|
|
|
|
options.row_cache = immutable_db_options.row_cache;
|
|
|
|
options.wal_filter = immutable_db_options.wal_filter;
|
|
|
|
options.fail_if_options_file_error =
|
|
|
|
immutable_db_options.fail_if_options_file_error;
|
|
|
|
options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
|
|
|
|
options.avoid_flush_during_recovery =
|
|
|
|
immutable_db_options.avoid_flush_during_recovery;
|
2016-11-02 22:22:13 +00:00
|
|
|
options.avoid_flush_during_shutdown =
|
|
|
|
mutable_db_options.avoid_flush_during_shutdown;
|
2022-01-28 21:26:32 +00:00
|
|
|
options.allow_ingest_behind = immutable_db_options.allow_ingest_behind;
|
2018-05-14 17:53:32 +00:00
|
|
|
options.two_write_queues = immutable_db_options.two_write_queues;
|
|
|
|
options.manual_wal_flush = immutable_db_options.manual_wal_flush;
|
2022-01-26 21:57:30 +00:00
|
|
|
options.wal_compression = immutable_db_options.wal_compression;
|
2024-09-26 21:36:29 +00:00
|
|
|
options.background_close_inactive_wals =
|
|
|
|
immutable_db_options.background_close_inactive_wals;
|
2018-11-12 20:22:10 +00:00
|
|
|
options.atomic_flush = immutable_db_options.atomic_flush;
|
2019-04-02 00:07:38 +00:00
|
|
|
options.avoid_unnecessary_blocking_io =
|
|
|
|
immutable_db_options.avoid_unnecessary_blocking_io;
|
2024-09-26 21:36:29 +00:00
|
|
|
options.write_dbid_to_manifest = immutable_db_options.write_dbid_to_manifest;
|
|
|
|
options.write_identity_file = immutable_db_options.write_identity_file;
|
|
|
|
options.prefix_seek_opt_in_only =
|
|
|
|
immutable_db_options.prefix_seek_opt_in_only;
|
2019-07-19 18:54:38 +00:00
|
|
|
options.log_readahead_size = immutable_db_options.log_readahead_size;
|
2020-03-29 22:57:02 +00:00
|
|
|
options.file_checksum_gen_factory =
|
|
|
|
immutable_db_options.file_checksum_gen_factory;
|
2020-03-21 02:17:54 +00:00
|
|
|
options.best_efforts_recovery = immutable_db_options.best_efforts_recovery;
|
2020-07-15 18:02:44 +00:00
|
|
|
options.max_bgerror_resume_count =
|
|
|
|
immutable_db_options.max_bgerror_resume_count;
|
|
|
|
options.bgerror_resume_retry_interval =
|
|
|
|
immutable_db_options.bgerror_resume_retry_interval;
|
2020-10-19 18:37:05 +00:00
|
|
|
options.db_host_id = immutable_db_options.db_host_id;
|
2020-11-13 06:08:03 +00:00
|
|
|
options.allow_data_in_errors = immutable_db_options.allow_data_in_errors;
|
2021-02-11 06:18:33 +00:00
|
|
|
options.checksum_handoff_file_types =
|
|
|
|
immutable_db_options.checksum_handoff_file_types;
|
2021-10-19 22:53:16 +00:00
|
|
|
options.lowest_used_cache_tier = immutable_db_options.lowest_used_cache_tier;
|
2022-05-20 23:48:50 +00:00
|
|
|
options.enforce_single_del_contracts =
|
|
|
|
immutable_db_options.enforce_single_del_contracts;
|
Offpeak in db option (#11893)
Summary:
RocksDB's primary function is to facilitate read and write operations. Compactions, while essential for minimizing read amplifications and optimizing storage, can sometimes compete with these primary tasks. Especially during periods of high read/write traffic, it's vital to ensure that primary operations receive priority, avoiding any potential disruptions or slowdowns. Conversely, during off-peak times when traffic is minimal, it's an opportune moment to tackle low-priority tasks like TTL based compactions, optimizing resource usage.
In this PR, we are incorporating the concept of off-peak time into RocksDB by introducing `daily_offpeak_time_utc` within the DBOptions. This setting is formatted as "HH:mm-HH:mm" where the first one before "-" is the start time and the second one is the end time, inclusive. It will be later used for resource optimization in subsequent PRs.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11893
Test Plan:
- New Unit Test Added - `DBOptionsTest::OffPeakTimes`
- Existing Unit Test Updated - `OptionsTest`, `OptionsSettableTest`
Reviewed By: pdillinger
Differential Revision: D49714553
Pulled By: jaykorean
fbshipit-source-id: fef51ea7c0fede6431c715bff116ddbb567c8752
2023-09-29 20:03:39 +00:00
|
|
|
options.daily_offpeak_time_utc = mutable_db_options.daily_offpeak_time_utc;
|
2024-08-24 02:49:25 +00:00
|
|
|
options.follower_refresh_catchup_period_ms =
|
|
|
|
immutable_db_options.follower_refresh_catchup_period_ms;
|
|
|
|
options.follower_catchup_retry_count =
|
|
|
|
immutable_db_options.follower_catchup_retry_count;
|
|
|
|
options.follower_catchup_retry_wait_ms =
|
|
|
|
immutable_db_options.follower_catchup_retry_wait_ms;
|
|
|
|
options.metadata_write_temperature =
|
|
|
|
immutable_db_options.metadata_write_temperature;
|
|
|
|
options.wal_write_temperature = immutable_db_options.wal_write_temperature;
|
2024-11-15 22:21:32 +00:00
|
|
|
options.compaction_service = immutable_db_options.compaction_service;
|
2016-09-23 23:34:04 +00:00
|
|
|
}
|
|
|
|
|
2016-09-15 05:10:28 +00:00
|
|
|
ColumnFamilyOptions BuildColumnFamilyOptions(
|
|
|
|
const ColumnFamilyOptions& options,
|
|
|
|
const MutableCFOptions& mutable_cf_options) {
|
|
|
|
ColumnFamilyOptions cf_opts(options);
|
2021-05-05 20:59:21 +00:00
|
|
|
UpdateColumnFamilyOptions(mutable_cf_options, &cf_opts);
|
2016-09-15 05:10:28 +00:00
|
|
|
// TODO(yhchiang): find some way to handle the following derived options
|
|
|
|
// * max_file_size
|
|
|
|
return cf_opts;
|
|
|
|
}
|
2021-04-23 03:42:50 +00:00
|
|
|
|
2021-05-05 20:59:21 +00:00
|
|
|
void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
|
|
|
|
ColumnFamilyOptions* cf_opts) {
|
2021-04-23 03:42:50 +00:00
|
|
|
// Memtable related options
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->write_buffer_size = moptions.write_buffer_size;
|
|
|
|
cf_opts->max_write_buffer_number = moptions.max_write_buffer_number;
|
|
|
|
cf_opts->arena_block_size = moptions.arena_block_size;
|
|
|
|
cf_opts->memtable_prefix_bloom_size_ratio =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.memtable_prefix_bloom_size_ratio;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->memtable_whole_key_filtering = moptions.memtable_whole_key_filtering;
|
|
|
|
cf_opts->memtable_huge_page_size = moptions.memtable_huge_page_size;
|
|
|
|
cf_opts->max_successive_merges = moptions.max_successive_merges;
|
2024-02-21 21:15:27 +00:00
|
|
|
cf_opts->strict_max_successive_merges = moptions.strict_max_successive_merges;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->inplace_update_num_locks = moptions.inplace_update_num_locks;
|
|
|
|
cf_opts->prefix_extractor = moptions.prefix_extractor;
|
2022-06-23 16:42:18 +00:00
|
|
|
cf_opts->experimental_mempurge_threshold =
|
|
|
|
moptions.experimental_mempurge_threshold;
|
2022-08-12 20:51:32 +00:00
|
|
|
cf_opts->memtable_protection_bytes_per_key =
|
|
|
|
moptions.memtable_protection_bytes_per_key;
|
2023-04-25 19:08:23 +00:00
|
|
|
cf_opts->block_protection_bytes_per_key =
|
|
|
|
moptions.block_protection_bytes_per_key;
|
2024-08-19 20:53:25 +00:00
|
|
|
cf_opts->paranoid_memory_checks = moptions.paranoid_memory_checks;
|
Delay bottommost level single file compactions (#11701)
Summary:
For leveled compaction, RocksDB has a special kind of compaction with reason "kBottommmostFiles" that compacts bottommost level files to clear data held by snapshots (more detail in https://github.com/facebook/rocksdb/issues/3009). Such compactions can happen soon after a relevant snapshot is released. For some use cases, a bottommost file may contain only a small amount of keys that can be cleared, so compacting such a file has a high write amp. In addition, these bottommost files may be compacted in compactions with reason other than "kBottommmostFiles" if we wait for some time (so that enough data is ingested to trigger such a compaction). This PR introduces an option `bottommost_file_compaction_delay` to specify the delay of these bottommost level single file compactions.
* The main change is in `VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction()` where we only add a file to `bottommost_files_marked_for_compaction_` if it oldest_snapshot is larger than its non-zero largest_seqno **and** the file is old enough. Note that if a file is not old enough but its largest_seqno is less than oldest_snapshot, we exclude it from the calculation of `bottommost_files_mark_threshold_`. This makes the change simpler, but such a file's eligibility for compaction will only be checked the next time `ComputeBottommostFilesMarkedForCompaction()` is called. This happens when a new Version is created (compaction, flush, SetOptions()...), a new enough snapshot is released (`VersionStorageInfo::UpdateOldestSnapshot()`) or when a compaction is picked and compaction score has to be re-calculated.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11701
Test Plan:
* Add two unit tests to test when bottommost_file_compaction_delay > 0.
* Ran crash test with the new option.
Reviewed By: jaykorean, ajkr
Differential Revision: D48331564
Pulled By: cbi42
fbshipit-source-id: c584f3dc5f6354fce3ed65f4c6366dc450b15ba8
2023-08-17 00:45:44 +00:00
|
|
|
cf_opts->bottommost_file_compaction_delay =
|
|
|
|
moptions.bottommost_file_compaction_delay;
|
2021-04-23 03:42:50 +00:00
|
|
|
|
|
|
|
// Compaction related options
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->disable_auto_compactions = moptions.disable_auto_compactions;
|
2024-10-17 21:13:20 +00:00
|
|
|
cf_opts->table_factory = moptions.table_factory;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->soft_pending_compaction_bytes_limit =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.soft_pending_compaction_bytes_limit;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->hard_pending_compaction_bytes_limit =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.hard_pending_compaction_bytes_limit;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->level0_file_num_compaction_trigger =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.level0_file_num_compaction_trigger;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->level0_slowdown_writes_trigger =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.level0_slowdown_writes_trigger;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->level0_stop_writes_trigger = moptions.level0_stop_writes_trigger;
|
|
|
|
cf_opts->max_compaction_bytes = moptions.max_compaction_bytes;
|
|
|
|
cf_opts->target_file_size_base = moptions.target_file_size_base;
|
|
|
|
cf_opts->target_file_size_multiplier = moptions.target_file_size_multiplier;
|
|
|
|
cf_opts->max_bytes_for_level_base = moptions.max_bytes_for_level_base;
|
|
|
|
cf_opts->max_bytes_for_level_multiplier =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.max_bytes_for_level_multiplier;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->ttl = moptions.ttl;
|
|
|
|
cf_opts->periodic_compaction_seconds = moptions.periodic_compaction_seconds;
|
2024-11-05 00:15:10 +00:00
|
|
|
cf_opts->preclude_last_level_data_seconds =
|
|
|
|
moptions.preclude_last_level_data_seconds;
|
|
|
|
cf_opts->preserve_internal_time_seconds =
|
|
|
|
moptions.preserve_internal_time_seconds;
|
2021-04-23 03:42:50 +00:00
|
|
|
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->max_bytes_for_level_multiplier_additional.clear();
|
2021-04-23 03:42:50 +00:00
|
|
|
for (auto value : moptions.max_bytes_for_level_multiplier_additional) {
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->max_bytes_for_level_multiplier_additional.emplace_back(value);
|
2021-04-23 03:42:50 +00:00
|
|
|
}
|
|
|
|
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->compaction_options_fifo = moptions.compaction_options_fifo;
|
|
|
|
cf_opts->compaction_options_universal = moptions.compaction_options_universal;
|
2021-04-23 03:42:50 +00:00
|
|
|
|
|
|
|
// Blob file related options
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->enable_blob_files = moptions.enable_blob_files;
|
|
|
|
cf_opts->min_blob_size = moptions.min_blob_size;
|
|
|
|
cf_opts->blob_file_size = moptions.blob_file_size;
|
|
|
|
cf_opts->blob_compression_type = moptions.blob_compression_type;
|
|
|
|
cf_opts->enable_blob_garbage_collection =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.enable_blob_garbage_collection;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->blob_garbage_collection_age_cutoff =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.blob_garbage_collection_age_cutoff;
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
2021-10-12 01:00:44 +00:00
|
|
|
cf_opts->blob_garbage_collection_force_threshold =
|
|
|
|
moptions.blob_garbage_collection_force_threshold;
|
2021-11-20 01:52:42 +00:00
|
|
|
cf_opts->blob_compaction_readahead_size =
|
|
|
|
moptions.blob_compaction_readahead_size;
|
2022-06-03 03:04:33 +00:00
|
|
|
cf_opts->blob_file_starting_level = moptions.blob_file_starting_level;
|
2022-07-17 14:13:59 +00:00
|
|
|
cf_opts->prepopulate_blob_cache = moptions.prepopulate_blob_cache;
|
2021-04-23 03:42:50 +00:00
|
|
|
|
|
|
|
// Misc options
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->max_sequential_skip_in_iterations =
|
2021-04-23 03:42:50 +00:00
|
|
|
moptions.max_sequential_skip_in_iterations;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->paranoid_file_checks = moptions.paranoid_file_checks;
|
|
|
|
cf_opts->report_bg_io_stats = moptions.report_bg_io_stats;
|
|
|
|
cf_opts->compression = moptions.compression;
|
|
|
|
cf_opts->compression_opts = moptions.compression_opts;
|
|
|
|
cf_opts->bottommost_compression = moptions.bottommost_compression;
|
|
|
|
cf_opts->bottommost_compression_opts = moptions.bottommost_compression_opts;
|
|
|
|
cf_opts->sample_for_compression = moptions.sample_for_compression;
|
2022-03-08 02:06:19 +00:00
|
|
|
cf_opts->compression_per_level = moptions.compression_per_level;
|
2022-08-08 21:36:34 +00:00
|
|
|
cf_opts->last_level_temperature = moptions.last_level_temperature;
|
2024-02-28 22:36:13 +00:00
|
|
|
cf_opts->default_write_temperature = moptions.default_write_temperature;
|
2023-08-03 02:58:56 +00:00
|
|
|
cf_opts->memtable_max_range_deletions = moptions.memtable_max_range_deletions;
|
Support pro-actively erasing obsolete block cache entries (#12694)
Summary:
Currently, when files become obsolete, the block cache entries associated with them just age out naturally. With pure LRU, this is not too bad, as once you "use" enough cache entries to (re-)fill the cache, you are guranteed to have purged the obsolete entries. However, HyperClockCache is a counting clock cache with a somewhat longer memory, so could be more negatively impacted by previously-hot cache entries becoming obsolete, and taking longer to age out than newer single-hit entries.
Part of the reason we still have this natural aging-out is that there's almost no connection between block cache entries and the file they are associated with. Everything is hashed into the same pool(s) of entries with nothing like a secondary index based on file. Keeping track of such an index could be expensive.
This change adds a new, mutable CF option `uncache_aggressiveness` for erasing obsolete block cache entries. The process can be speculative, lossy, or unproductive because not all potential block cache entries associated with files will be resident in memory, and attempting to remove them all could be wasted CPU time. Rather than a simple on/off switch, `uncache_aggressiveness` basically tells RocksDB how much CPU you're willing to burn trying to purge obsolete block cache entries. When such efforts are not sufficiently productive for a file, we stop and move on.
The option is in ColumnFamilyOptions so that it is dynamically changeable for already-open files, and customizeable by CF.
Note that this block cache removal happens as part of the process of purging obsolete files, which is often in a background thread (depending on `background_purge_on_iterator_cleanup` and `avoid_unnecessary_blocking_io` options) rather than along CPU critical paths.
Notable auxiliary code details:
* Possibly fixing some issues with trivial moves with `only_delete_metadata`: unnecessary TableCache::Evict in that case and missing from the ObsoleteFileInfo move operator. (Not able to reproduce an current failure.)
* Remove suspicious TableCache::Erase() from VersionSet::AddObsoleteBlobFile() (TODO follow-up item)
Marked EXPERIMENTAL until more thorough validation is complete.
Direct stats of this functionality are omitted because they could be misleading. Block cache hit rate is a better indicator of benefit, and CPU profiling a better indicator of cost.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12694
Test Plan:
* Unit tests added, including refactoring an existing test to make better use of parameterized tests.
* Added to crash test.
* Performance, sample command:
```
for I in `seq 1 10`; do for UA in 300; do for CT in lru_cache fixed_hyper_clock_cache auto_hyper_clock_cache; do rm -rf /dev/shm/test3; TEST_TMPDIR=/dev/shm/test3 /usr/bin/time ./db_bench -benchmarks=readwhilewriting -num=13000000 -read_random_exp_range=6 -write_buffer_size=10000000 -bloom_bits=10 -cache_type=$CT -cache_size=390000000 -cache_index_and_filter_blocks=1 -disable_wal=1 -duration=60 -statistics -uncache_aggressiveness=$UA 2>&1 | grep -E 'micros/op|rocksdb.block.cache.data.(hit|miss)|rocksdb.number.keys.(read|written)|maxresident' | awk '/rocksdb.block.cache.data.miss/ { miss = $4 } /rocksdb.block.cache.data.hit/ { hit = $4 } { print } END { print "hit rate = " ((hit * 1.0) / (miss + hit)) }' | tee -a results-$CT-$UA; done; done; done
```
Averaging 10 runs each case, block cache data block hit rates
```
lru_cache
UA=0 -> hit rate = 0.327, ops/s = 87668, user CPU sec = 139.0
UA=300 -> hit rate = 0.336, ops/s = 87960, user CPU sec = 139.0
fixed_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 100069, user CPU sec = 139.9
UA=300 -> hit rate = 0.343, ops/s = 100104, user CPU sec = 140.2
auto_hyper_clock_cache
UA=0 -> hit rate = 0.336, ops/s = 97580, user CPU sec = 140.5
UA=300 -> hit rate = 0.345, ops/s = 97972, user CPU sec = 139.8
```
Conclusion: up to roughly 1 percentage point of improved block cache hit rate, likely leading to overall improved efficiency (because the foreground CPU cost of cache misses likely outweighs the background CPU cost of erasure, let alone I/O savings).
Reviewed By: ajkr
Differential Revision: D57932442
Pulled By: pdillinger
fbshipit-source-id: 84a243ca5f965f731f346a4853009780a904af6c
2024-06-07 15:57:11 +00:00
|
|
|
cf_opts->uncache_aggressiveness = moptions.uncache_aggressiveness;
|
2021-05-05 20:59:21 +00:00
|
|
|
}
|
2021-04-23 03:42:50 +00:00
|
|
|
|
2021-05-05 20:59:21 +00:00
|
|
|
void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,
|
|
|
|
ColumnFamilyOptions* cf_opts) {
|
|
|
|
cf_opts->compaction_style = ioptions.compaction_style;
|
|
|
|
cf_opts->compaction_pri = ioptions.compaction_pri;
|
|
|
|
cf_opts->comparator = ioptions.user_comparator;
|
|
|
|
cf_opts->merge_operator = ioptions.merge_operator;
|
|
|
|
cf_opts->compaction_filter = ioptions.compaction_filter;
|
|
|
|
cf_opts->compaction_filter_factory = ioptions.compaction_filter_factory;
|
|
|
|
cf_opts->min_write_buffer_number_to_merge =
|
2021-04-23 03:42:50 +00:00
|
|
|
ioptions.min_write_buffer_number_to_merge;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->max_write_buffer_number_to_maintain =
|
2021-04-23 03:42:50 +00:00
|
|
|
ioptions.max_write_buffer_number_to_maintain;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->max_write_buffer_size_to_maintain =
|
2021-04-23 03:42:50 +00:00
|
|
|
ioptions.max_write_buffer_size_to_maintain;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->inplace_update_support = ioptions.inplace_update_support;
|
|
|
|
cf_opts->inplace_callback = ioptions.inplace_callback;
|
|
|
|
cf_opts->memtable_factory = ioptions.memtable_factory;
|
|
|
|
cf_opts->table_properties_collector_factories =
|
2021-04-23 03:42:50 +00:00
|
|
|
ioptions.table_properties_collector_factories;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->bloom_locality = ioptions.bloom_locality;
|
|
|
|
cf_opts->level_compaction_dynamic_level_bytes =
|
2021-04-23 03:42:50 +00:00
|
|
|
ioptions.level_compaction_dynamic_level_bytes;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->num_levels = ioptions.num_levels;
|
|
|
|
cf_opts->optimize_filters_for_hits = ioptions.optimize_filters_for_hits;
|
|
|
|
cf_opts->force_consistency_checks = ioptions.force_consistency_checks;
|
|
|
|
cf_opts->memtable_insert_with_hint_prefix_extractor =
|
2021-04-23 03:42:50 +00:00
|
|
|
ioptions.memtable_insert_with_hint_prefix_extractor;
|
2021-05-05 20:59:21 +00:00
|
|
|
cf_opts->cf_paths = ioptions.cf_paths;
|
|
|
|
cf_opts->compaction_thread_limiter = ioptions.compaction_thread_limiter;
|
|
|
|
cf_opts->sst_partitioner_factory = ioptions.sst_partitioner_factory;
|
2022-06-14 21:19:26 +00:00
|
|
|
cf_opts->blob_cache = ioptions.blob_cache;
|
2023-04-12 00:50:34 +00:00
|
|
|
cf_opts->persist_user_defined_timestamps =
|
|
|
|
ioptions.persist_user_defined_timestamps;
|
2023-08-18 00:06:57 +00:00
|
|
|
cf_opts->default_temperature = ioptions.default_temperature;
|
2021-04-23 03:42:50 +00:00
|
|
|
|
|
|
|
// TODO(yhchiang): find some way to handle the following derived options
|
|
|
|
// * max_file_size
|
|
|
|
}
|
2016-09-15 05:10:28 +00:00
|
|
|
|
2017-11-18 01:02:13 +00:00
|
|
|
std::map<CompactionStyle, std::string>
|
|
|
|
OptionsHelper::compaction_style_to_string = {
|
|
|
|
{kCompactionStyleLevel, "kCompactionStyleLevel"},
|
|
|
|
{kCompactionStyleUniversal, "kCompactionStyleUniversal"},
|
|
|
|
{kCompactionStyleFIFO, "kCompactionStyleFIFO"},
|
|
|
|
{kCompactionStyleNone, "kCompactionStyleNone"}};
|
|
|
|
|
|
|
|
std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = {
|
|
|
|
{kByCompensatedSize, "kByCompensatedSize"},
|
|
|
|
{kOldestLargestSeqFirst, "kOldestLargestSeqFirst"},
|
|
|
|
{kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"},
|
2022-06-21 18:56:53 +00:00
|
|
|
{kMinOverlappingRatio, "kMinOverlappingRatio"},
|
|
|
|
{kRoundRobin, "kRoundRobin"}};
|
2017-11-18 01:02:13 +00:00
|
|
|
|
|
|
|
std::map<CompactionStopStyle, std::string>
|
|
|
|
OptionsHelper::compaction_stop_style_to_string = {
|
|
|
|
{kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"},
|
|
|
|
{kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}};
|
|
|
|
|
New backup meta schema, with file temperatures (#9660)
Summary:
The primary goal of this change is to add support for backing up and
restoring (applying on restore) file temperature metadata, without
committing to either the DB manifest or the FS reported "current"
temperatures being exclusive "source of truth".
To achieve this goal, we need to add temperature information to backup
metadata, which requires updated backup meta schema. Fortunately I
prepared for this in https://github.com/facebook/rocksdb/issues/8069, which began forward compatibility in version
6.19.0 for this kind of schema update. (Previously, backup meta schema
was not extensible! Making this schema update public will allow some
other "nice to have" features like taking backups with hard links, and
avoiding crc32c checksum computation when another checksum is already
available.) While schema version 2 is newly public, the default schema
version is still 1. Until we change the default, users will need to set
to 2 to enable features like temperature data backup+restore. New
metadata like temperature information will be ignored with a warning
in versions before this change and since 6.19.0. The metadata is
considered ignorable because a functioning DB can be restored without
it.
Some detail:
* Some renaming because "future schema" is now just public schema 2.
* Initialize some atomics in TestFs (linter reported)
* Add temperature hint support to SstFileDumper (used by BackupEngine)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9660
Test Plan:
related unit test majorly updated for the new functionality,
including some shared testing support for tracking temperatures in a FS.
Some other tests and testing hooks into production code also updated for
making the backup meta schema change public.
Reviewed By: ajkr
Differential Revision: D34686968
Pulled By: pdillinger
fbshipit-source-id: 3ac1fa3e67ee97ca8a5103d79cc87d872c1d862a
2022-03-18 18:06:17 +00:00
|
|
|
std::map<Temperature, std::string> OptionsHelper::temperature_to_string = {
|
|
|
|
{Temperature::kUnknown, "kUnknown"},
|
|
|
|
{Temperature::kHot, "kHot"},
|
|
|
|
{Temperature::kWarm, "kWarm"},
|
|
|
|
{Temperature::kCold, "kCold"}};
|
|
|
|
|
2017-11-18 01:02:13 +00:00
|
|
|
std::unordered_map<std::string, ChecksumType>
|
|
|
|
OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum},
|
|
|
|
{"kCRC32c", kCRC32c},
|
2018-11-01 22:39:40 +00:00
|
|
|
{"kxxHash", kxxHash},
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
{"kxxHash64", kxxHash64},
|
|
|
|
{"kXXH3", kXXH3}};
|
2017-11-18 01:02:13 +00:00
|
|
|
|
2018-01-23 22:36:54 +00:00
|
|
|
std::unordered_map<std::string, CompressionType>
|
|
|
|
OptionsHelper::compression_type_string_map = {
|
|
|
|
{"kNoCompression", kNoCompression},
|
|
|
|
{"kSnappyCompression", kSnappyCompression},
|
|
|
|
{"kZlibCompression", kZlibCompression},
|
|
|
|
{"kBZip2Compression", kBZip2Compression},
|
|
|
|
{"kLZ4Compression", kLZ4Compression},
|
|
|
|
{"kLZ4HCCompression", kLZ4HCCompression},
|
|
|
|
{"kXpressCompression", kXpressCompression},
|
|
|
|
{"kZSTD", kZSTD},
|
|
|
|
{"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
|
|
|
|
{"kDisableCompressionOption", kDisableCompressionOption}};
|
2020-04-27 23:51:42 +00:00
|
|
|
|
|
|
|
std::vector<CompressionType> GetSupportedCompressions() {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
// std::set internally to deduplicate potential name aliases
|
|
|
|
std::set<CompressionType> supported_compressions;
|
2020-04-27 23:51:42 +00:00
|
|
|
for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) {
|
|
|
|
CompressionType t = comp_to_name.second;
|
|
|
|
if (t != kDisableCompressionOption && CompressionTypeSupported(t)) {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
supported_compressions.insert(t);
|
2020-04-27 23:51:42 +00:00
|
|
|
}
|
|
|
|
}
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
return std::vector<CompressionType>(supported_compressions.begin(),
|
|
|
|
supported_compressions.end());
|
2020-04-27 23:51:42 +00:00
|
|
|
}
|
|
|
|
|
2020-11-03 03:20:15 +00:00
|
|
|
std::vector<CompressionType> GetSupportedDictCompressions() {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
std::set<CompressionType> dict_compression_types;
|
2020-11-03 03:20:15 +00:00
|
|
|
for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) {
|
|
|
|
CompressionType t = comp_to_name.second;
|
|
|
|
if (t != kDisableCompressionOption && DictCompressionTypeSupported(t)) {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
dict_compression_types.insert(t);
|
2020-11-03 03:20:15 +00:00
|
|
|
}
|
|
|
|
}
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
2021-10-29 05:13:47 +00:00
|
|
|
return std::vector<CompressionType>(dict_compression_types.begin(),
|
|
|
|
dict_compression_types.end());
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<ChecksumType> GetSupportedChecksums() {
|
|
|
|
std::set<ChecksumType> checksum_types;
|
|
|
|
for (const auto& e : OptionsHelper::checksum_type_string_map) {
|
|
|
|
checksum_types.insert(e.second);
|
|
|
|
}
|
|
|
|
return std::vector<ChecksumType>(checksum_types.begin(),
|
|
|
|
checksum_types.end());
|
2020-11-03 03:20:15 +00:00
|
|
|
}
|
|
|
|
|
2021-05-13 21:28:50 +00:00
|
|
|
static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
|
2020-09-14 23:59:00 +00:00
|
|
|
const std::string& value) {
|
2015-08-26 23:13:56 +00:00
|
|
|
switch (opt_type) {
|
|
|
|
case OptionType::kBoolean:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<bool*>(opt_address) = ParseBoolean("", value);
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kInt:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<int*>(opt_address) = ParseInt(value);
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
2019-03-12 20:46:12 +00:00
|
|
|
case OptionType::kInt32T:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<int32_t*>(opt_address) = ParseInt32(value);
|
2019-03-12 20:46:12 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kInt64T:
|
2021-05-13 21:28:50 +00:00
|
|
|
PutUnaligned(static_cast<int64_t*>(opt_address), ParseInt64(value));
|
2019-03-12 20:46:12 +00:00
|
|
|
break;
|
2015-08-26 23:13:56 +00:00
|
|
|
case OptionType::kUInt:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<unsigned int*>(opt_address) = ParseUint32(value);
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
2021-05-20 04:40:43 +00:00
|
|
|
case OptionType::kUInt8T:
|
|
|
|
*static_cast<uint8_t*>(opt_address) = ParseUint8(value);
|
|
|
|
break;
|
2015-08-26 23:13:56 +00:00
|
|
|
case OptionType::kUInt32T:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<uint32_t*>(opt_address) = ParseUint32(value);
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kUInt64T:
|
2021-05-13 21:28:50 +00:00
|
|
|
PutUnaligned(static_cast<uint64_t*>(opt_address), ParseUint64(value));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kSizeT:
|
2021-05-13 21:28:50 +00:00
|
|
|
PutUnaligned(static_cast<size_t*>(opt_address), ParseSizeT(value));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
2023-09-15 22:46:10 +00:00
|
|
|
case OptionType::kAtomicInt:
|
|
|
|
static_cast<std::atomic<int>*>(opt_address)
|
|
|
|
->store(ParseInt(value), std::memory_order_release);
|
|
|
|
break;
|
2015-08-26 23:13:56 +00:00
|
|
|
case OptionType::kString:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<std::string*>(opt_address) = value;
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kDouble:
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<double*>(opt_address) = ParseDouble(value);
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kCompactionStyle:
|
2015-11-18 00:41:54 +00:00
|
|
|
return ParseEnum<CompactionStyle>(
|
|
|
|
compaction_style_string_map, value,
|
2021-05-13 21:28:50 +00:00
|
|
|
static_cast<CompactionStyle*>(opt_address));
|
2017-03-02 18:08:49 +00:00
|
|
|
case OptionType::kCompactionPri:
|
2021-05-13 21:28:50 +00:00
|
|
|
return ParseEnum<CompactionPri>(compaction_pri_string_map, value,
|
|
|
|
static_cast<CompactionPri*>(opt_address));
|
2015-10-02 22:35:32 +00:00
|
|
|
case OptionType::kCompressionType:
|
2015-11-18 00:41:54 +00:00
|
|
|
return ParseEnum<CompressionType>(
|
|
|
|
compression_type_string_map, value,
|
2021-05-13 21:28:50 +00:00
|
|
|
static_cast<CompressionType*>(opt_address));
|
2015-10-11 19:17:42 +00:00
|
|
|
case OptionType::kChecksumType:
|
2021-05-13 21:28:50 +00:00
|
|
|
return ParseEnum<ChecksumType>(checksum_type_string_map, value,
|
|
|
|
static_cast<ChecksumType*>(opt_address));
|
2015-10-28 17:46:01 +00:00
|
|
|
case OptionType::kEncodingType:
|
2021-05-13 21:28:50 +00:00
|
|
|
return ParseEnum<EncodingType>(encoding_type_string_map, value,
|
|
|
|
static_cast<EncodingType*>(opt_address));
|
2017-12-11 21:12:12 +00:00
|
|
|
case OptionType::kCompactionStopStyle:
|
|
|
|
return ParseEnum<CompactionStopStyle>(
|
|
|
|
compaction_stop_style_string_map, value,
|
2021-05-13 21:28:50 +00:00
|
|
|
static_cast<CompactionStopStyle*>(opt_address));
|
2021-05-12 19:34:22 +00:00
|
|
|
case OptionType::kEncodedString: {
|
2021-05-13 21:28:50 +00:00
|
|
|
std::string* output_addr = static_cast<std::string*>(opt_address);
|
2021-05-12 19:34:22 +00:00
|
|
|
(Slice(value)).DecodeHex(output_addr);
|
|
|
|
break;
|
|
|
|
}
|
2022-01-25 22:58:48 +00:00
|
|
|
case OptionType::kTemperature: {
|
|
|
|
return ParseEnum<Temperature>(temperature_string_map, value,
|
|
|
|
static_cast<Temperature*>(opt_address));
|
|
|
|
}
|
2015-08-26 23:13:56 +00:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2021-05-13 21:28:50 +00:00
|
|
|
bool SerializeSingleOptionHelper(const void* opt_address,
|
2015-08-26 23:13:56 +00:00
|
|
|
const OptionType opt_type,
|
|
|
|
std::string* value) {
|
|
|
|
assert(value);
|
|
|
|
switch (opt_type) {
|
|
|
|
case OptionType::kBoolean:
|
2021-05-13 21:28:50 +00:00
|
|
|
*value = *(static_cast<const bool*>(opt_address)) ? "true" : "false";
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kInt:
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(*(static_cast<const int*>(opt_address)));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
2019-03-12 20:46:12 +00:00
|
|
|
case OptionType::kInt32T:
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(*(static_cast<const int32_t*>(opt_address)));
|
2019-03-12 20:46:12 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kInt64T:
|
|
|
|
{
|
|
|
|
int64_t v;
|
2021-05-13 21:28:50 +00:00
|
|
|
GetUnaligned(static_cast<const int64_t*>(opt_address), &v);
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(v);
|
2019-03-12 20:46:12 +00:00
|
|
|
}
|
|
|
|
break;
|
2015-08-26 23:13:56 +00:00
|
|
|
case OptionType::kUInt:
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(*(static_cast<const unsigned int*>(opt_address)));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
2021-05-20 04:40:43 +00:00
|
|
|
case OptionType::kUInt8T:
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(*(static_cast<const uint8_t*>(opt_address)));
|
2021-05-20 04:40:43 +00:00
|
|
|
break;
|
2015-08-26 23:13:56 +00:00
|
|
|
case OptionType::kUInt32T:
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(*(static_cast<const uint32_t*>(opt_address)));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kUInt64T:
|
2017-04-22 03:41:37 +00:00
|
|
|
{
|
|
|
|
uint64_t v;
|
2021-05-13 21:28:50 +00:00
|
|
|
GetUnaligned(static_cast<const uint64_t*>(opt_address), &v);
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(v);
|
2017-04-22 03:41:37 +00:00
|
|
|
}
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kSizeT:
|
2017-04-22 03:41:37 +00:00
|
|
|
{
|
|
|
|
size_t v;
|
2021-05-13 21:28:50 +00:00
|
|
|
GetUnaligned(static_cast<const size_t*>(opt_address), &v);
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(v);
|
2017-04-22 03:41:37 +00:00
|
|
|
}
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kDouble:
|
2022-05-06 20:03:58 +00:00
|
|
|
*value = std::to_string(*(static_cast<const double*>(opt_address)));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
2023-09-15 22:46:10 +00:00
|
|
|
case OptionType::kAtomicInt:
|
|
|
|
*value = std::to_string(static_cast<const std::atomic<int>*>(opt_address)
|
|
|
|
->load(std::memory_order_acquire));
|
|
|
|
break;
|
2015-08-26 23:13:56 +00:00
|
|
|
case OptionType::kString:
|
2021-05-13 21:28:50 +00:00
|
|
|
*value =
|
|
|
|
EscapeOptionString(*(static_cast<const std::string*>(opt_address)));
|
2015-08-26 23:13:56 +00:00
|
|
|
break;
|
|
|
|
case OptionType::kCompactionStyle:
|
2015-11-18 00:41:54 +00:00
|
|
|
return SerializeEnum<CompactionStyle>(
|
|
|
|
compaction_style_string_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
*(static_cast<const CompactionStyle*>(opt_address)), value);
|
2017-03-02 18:08:49 +00:00
|
|
|
case OptionType::kCompactionPri:
|
|
|
|
return SerializeEnum<CompactionPri>(
|
|
|
|
compaction_pri_string_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
*(static_cast<const CompactionPri*>(opt_address)), value);
|
2015-10-02 22:35:32 +00:00
|
|
|
case OptionType::kCompressionType:
|
2015-11-18 00:41:54 +00:00
|
|
|
return SerializeEnum<CompressionType>(
|
|
|
|
compression_type_string_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
*(static_cast<const CompressionType*>(opt_address)), value);
|
2015-10-02 22:35:32 +00:00
|
|
|
break;
|
2015-10-11 19:17:42 +00:00
|
|
|
case OptionType::kChecksumType:
|
2015-11-18 00:41:54 +00:00
|
|
|
return SerializeEnum<ChecksumType>(
|
|
|
|
checksum_type_string_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<const ChecksumType*>(opt_address), value);
|
2015-10-28 17:46:01 +00:00
|
|
|
case OptionType::kEncodingType:
|
2015-11-18 00:41:54 +00:00
|
|
|
return SerializeEnum<EncodingType>(
|
|
|
|
encoding_type_string_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<const EncodingType*>(opt_address), value);
|
2017-12-11 21:12:12 +00:00
|
|
|
case OptionType::kCompactionStopStyle:
|
|
|
|
return SerializeEnum<CompactionStopStyle>(
|
|
|
|
compaction_stop_style_string_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
*static_cast<const CompactionStopStyle*>(opt_address), value);
|
2021-05-12 19:34:22 +00:00
|
|
|
case OptionType::kEncodedString: {
|
2021-05-13 21:28:50 +00:00
|
|
|
const auto* ptr = static_cast<const std::string*>(opt_address);
|
2021-05-12 19:34:22 +00:00
|
|
|
*value = (Slice(*ptr)).ToString(true);
|
|
|
|
break;
|
|
|
|
}
|
2022-01-25 22:58:48 +00:00
|
|
|
case OptionType::kTemperature: {
|
|
|
|
return SerializeEnum<Temperature>(
|
|
|
|
temperature_string_map, *static_cast<const Temperature*>(opt_address),
|
|
|
|
value);
|
|
|
|
}
|
2015-08-26 23:13:56 +00:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
template <typename T>
|
|
|
|
Status ConfigureFromMap(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opt_map,
|
|
|
|
const std::string& option_name, Configurable* config, T* new_opts) {
|
|
|
|
Status s = config->ConfigureFromMap(config_options, opt_map);
|
|
|
|
if (s.ok()) {
|
|
|
|
*new_opts = *(config->GetOptions<T>(option_name));
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2016-10-14 19:25:39 +00:00
|
|
|
|
2014-12-22 21:18:57 +00:00
|
|
|
Status StringToMap(const std::string& opts_str,
|
|
|
|
std::unordered_map<std::string, std::string>* opts_map) {
|
2014-10-10 17:00:12 +00:00
|
|
|
assert(opts_map);
|
|
|
|
// Example:
|
2014-12-22 21:18:57 +00:00
|
|
|
// opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
|
|
|
|
// "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
|
2014-10-10 17:00:12 +00:00
|
|
|
size_t pos = 0;
|
|
|
|
std::string opts = trim(opts_str);
|
2020-05-21 17:56:40 +00:00
|
|
|
// If the input string starts and ends with "{...}", strip off the brackets
|
|
|
|
while (opts.size() > 2 && opts[0] == '{' && opts[opts.size() - 1] == '}') {
|
|
|
|
opts = trim(opts.substr(1, opts.size() - 2));
|
|
|
|
}
|
|
|
|
|
2014-10-10 17:00:12 +00:00
|
|
|
while (pos < opts.size()) {
|
2021-07-16 22:04:29 +00:00
|
|
|
size_t eq_pos = opts.find_first_of("={};", pos);
|
2014-10-10 17:00:12 +00:00
|
|
|
if (eq_pos == std::string::npos) {
|
2014-12-22 21:18:57 +00:00
|
|
|
return Status::InvalidArgument("Mismatched key value pair, '=' expected");
|
2021-07-16 22:04:29 +00:00
|
|
|
} else if (opts[eq_pos] != '=') {
|
|
|
|
return Status::InvalidArgument("Unexpected char in key");
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
2021-07-16 22:04:29 +00:00
|
|
|
|
2014-10-10 17:00:12 +00:00
|
|
|
std::string key = trim(opts.substr(pos, eq_pos - pos));
|
2014-12-22 21:18:57 +00:00
|
|
|
if (key.empty()) {
|
|
|
|
return Status::InvalidArgument("Empty key found");
|
|
|
|
}
|
2014-10-10 17:00:12 +00:00
|
|
|
|
2020-06-03 19:19:54 +00:00
|
|
|
std::string value;
|
|
|
|
Status s = OptionTypeInfo::NextToken(opts, ';', eq_pos + 1, &pos, &value);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
2014-10-10 17:00:12 +00:00
|
|
|
} else {
|
2020-06-03 19:19:54 +00:00
|
|
|
(*opts_map)[key] = value;
|
|
|
|
if (pos == std::string::npos) {
|
2014-12-22 21:18:57 +00:00
|
|
|
break;
|
|
|
|
} else {
|
2020-06-03 19:19:54 +00:00
|
|
|
pos++;
|
2014-12-22 21:18:57 +00:00
|
|
|
}
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-12-22 21:18:57 +00:00
|
|
|
return Status::OK();
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
|
|
|
|
2015-08-18 20:30:18 +00:00
|
|
|
|
2017-10-19 22:19:20 +00:00
|
|
|
Status GetStringFromDBOptions(std::string* opt_string,
|
|
|
|
const DBOptions& db_options,
|
|
|
|
const std::string& delimiter) {
|
2021-05-11 13:45:49 +00:00
|
|
|
ConfigOptions config_options(db_options);
|
2020-04-22 00:35:28 +00:00
|
|
|
config_options.delimiter = delimiter;
|
|
|
|
return GetStringFromDBOptions(config_options, db_options, opt_string);
|
|
|
|
}
|
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
Status GetStringFromDBOptions(const ConfigOptions& config_options,
|
2020-04-22 00:35:28 +00:00
|
|
|
const DBOptions& db_options,
|
|
|
|
std::string* opt_string) {
|
2020-09-14 23:59:00 +00:00
|
|
|
assert(opt_string);
|
|
|
|
opt_string->clear();
|
|
|
|
auto config = DBOptionsAsConfigurable(db_options);
|
|
|
|
return config->GetOptionString(config_options, opt_string);
|
|
|
|
}
|
|
|
|
|
2015-08-26 23:13:56 +00:00
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 21:42:40 +00:00
|
|
|
Status GetStringFromColumnFamilyOptions(std::string* opt_string,
|
|
|
|
const ColumnFamilyOptions& cf_options,
|
|
|
|
const std::string& delimiter) {
|
2020-04-22 00:35:28 +00:00
|
|
|
ConfigOptions config_options;
|
|
|
|
config_options.delimiter = delimiter;
|
|
|
|
return GetStringFromColumnFamilyOptions(config_options, cf_options,
|
|
|
|
opt_string);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
|
|
|
|
const ColumnFamilyOptions& cf_options,
|
|
|
|
std::string* opt_string) {
|
2020-09-14 23:59:00 +00:00
|
|
|
const auto config = CFOptionsAsConfigurable(cf_options);
|
|
|
|
return config->GetOptionString(config_options, opt_string);
|
2015-08-26 23:13:56 +00:00
|
|
|
}
|
|
|
|
|
2016-05-18 22:03:21 +00:00
|
|
|
Status GetStringFromCompressionType(std::string* compression_str,
|
|
|
|
CompressionType compression_type) {
|
|
|
|
bool ok = SerializeEnum<CompressionType>(compression_type_string_map,
|
|
|
|
compression_type, compression_str);
|
|
|
|
if (ok) {
|
|
|
|
return Status::OK();
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument("Invalid compression types");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-22 00:35:28 +00:00
|
|
|
Status GetColumnFamilyOptionsFromMap(
|
|
|
|
const ConfigOptions& config_options,
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
2015-11-12 14:52:43 +00:00
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
2020-04-22 00:35:28 +00:00
|
|
|
ColumnFamilyOptions* new_options) {
|
2014-09-17 19:46:32 +00:00
|
|
|
assert(new_options);
|
2020-09-14 23:59:00 +00:00
|
|
|
|
2014-09-17 19:46:32 +00:00
|
|
|
*new_options = base_options;
|
2020-09-14 23:59:00 +00:00
|
|
|
|
|
|
|
const auto config = CFOptionsAsConfigurable(base_options);
|
2020-10-20 18:51:51 +00:00
|
|
|
Status s = ConfigureFromMap<ColumnFamilyOptions>(
|
|
|
|
config_options, opts_map, OptionsHelper::kCFOptionsName, config.get(),
|
|
|
|
new_options);
|
|
|
|
// Translate any errors (NotFound, NotSupported, to InvalidArgument
|
|
|
|
if (s.ok() || s.IsInvalidArgument()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(s.getState());
|
|
|
|
}
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
|
|
|
|
2020-04-22 00:35:28 +00:00
|
|
|
Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options,
|
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::string& opts_str,
|
|
|
|
ColumnFamilyOptions* new_options) {
|
2014-10-10 17:00:12 +00:00
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
2014-12-22 21:18:57 +00:00
|
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!s.ok()) {
|
2016-08-11 21:54:29 +00:00
|
|
|
*new_options = base_options;
|
2014-12-22 21:18:57 +00:00
|
|
|
return s;
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
2020-04-22 00:35:28 +00:00
|
|
|
return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
|
|
|
|
new_options);
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
|
|
|
|
2020-04-22 00:35:28 +00:00
|
|
|
Status GetDBOptionsFromMap(
|
|
|
|
const ConfigOptions& config_options, const DBOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
DBOptions* new_options) {
|
2014-10-10 17:00:12 +00:00
|
|
|
assert(new_options);
|
|
|
|
*new_options = base_options;
|
2020-09-14 23:59:00 +00:00
|
|
|
auto config = DBOptionsAsConfigurable(base_options);
|
2020-10-20 18:51:51 +00:00
|
|
|
Status s = ConfigureFromMap<DBOptions>(config_options, opts_map,
|
|
|
|
OptionsHelper::kDBOptionsName,
|
|
|
|
config.get(), new_options);
|
|
|
|
// Translate any errors (NotFound, NotSupported, to InvalidArgument
|
|
|
|
if (s.ok() || s.IsInvalidArgument()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(s.getState());
|
|
|
|
}
|
2014-09-17 19:46:32 +00:00
|
|
|
}
|
|
|
|
|
2020-04-22 00:35:28 +00:00
|
|
|
Status GetDBOptionsFromString(const ConfigOptions& config_options,
|
|
|
|
const DBOptions& base_options,
|
|
|
|
const std::string& opts_str,
|
|
|
|
DBOptions* new_options) {
|
2014-10-10 17:00:12 +00:00
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
2014-12-22 21:18:57 +00:00
|
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!s.ok()) {
|
2016-08-11 21:54:29 +00:00
|
|
|
*new_options = base_options;
|
2014-12-22 21:18:57 +00:00
|
|
|
return s;
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
2020-04-22 00:35:28 +00:00
|
|
|
return GetDBOptionsFromMap(config_options, base_options, opts_map,
|
|
|
|
new_options);
|
2014-10-10 17:00:12 +00:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:11:14 +00:00
|
|
|
Status GetOptionsFromString(const Options& base_options,
|
|
|
|
const std::string& opts_str, Options* new_options) {
|
2021-05-11 13:45:49 +00:00
|
|
|
ConfigOptions config_options(base_options);
|
2020-04-22 00:35:28 +00:00
|
|
|
config_options.input_strings_escaped = false;
|
|
|
|
config_options.ignore_unknown_options = false;
|
|
|
|
|
|
|
|
return GetOptionsFromString(config_options, base_options, opts_str,
|
|
|
|
new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetOptionsFromString(const ConfigOptions& config_options,
|
|
|
|
const Options& base_options,
|
|
|
|
const std::string& opts_str, Options* new_options) {
|
2020-09-14 23:59:00 +00:00
|
|
|
ColumnFamilyOptions new_cf_options;
|
|
|
|
std::unordered_map<std::string, std::string> unused_opts;
|
2015-02-20 03:11:14 +00:00
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
2020-09-14 23:59:00 +00:00
|
|
|
|
2021-05-11 13:45:49 +00:00
|
|
|
assert(new_options);
|
2020-09-14 23:59:00 +00:00
|
|
|
*new_options = base_options;
|
2015-02-20 03:11:14 +00:00
|
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
2020-09-14 23:59:00 +00:00
|
|
|
auto config = DBOptionsAsConfigurable(base_options);
|
|
|
|
s = config->ConfigureFromMap(config_options, opts_map, &unused_opts);
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
DBOptions* new_db_options =
|
|
|
|
config->GetOptions<DBOptions>(OptionsHelper::kDBOptionsName);
|
|
|
|
if (!unused_opts.empty()) {
|
|
|
|
s = GetColumnFamilyOptionsFromMap(config_options, base_options,
|
|
|
|
unused_opts, &new_cf_options);
|
|
|
|
if (s.ok()) {
|
|
|
|
*new_options = Options(*new_db_options, new_cf_options);
|
|
|
|
}
|
2015-02-20 03:11:14 +00:00
|
|
|
} else {
|
2020-09-14 23:59:00 +00:00
|
|
|
*new_options = Options(*new_db_options, base_options);
|
2015-10-28 17:46:01 +00:00
|
|
|
}
|
2015-10-11 19:17:42 +00:00
|
|
|
}
|
2020-10-20 18:51:51 +00:00
|
|
|
// Translate any errors (NotFound, NotSupported, to InvalidArgument
|
|
|
|
if (s.ok() || s.IsInvalidArgument()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(s.getState());
|
|
|
|
}
|
2015-10-11 19:17:42 +00:00
|
|
|
}
|
|
|
|
|
2017-11-18 01:02:13 +00:00
|
|
|
std::unordered_map<std::string, EncodingType>
|
|
|
|
OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain},
|
|
|
|
{"kPrefix", kPrefix}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, CompactionStyle>
|
|
|
|
OptionsHelper::compaction_style_string_map = {
|
|
|
|
{"kCompactionStyleLevel", kCompactionStyleLevel},
|
|
|
|
{"kCompactionStyleUniversal", kCompactionStyleUniversal},
|
|
|
|
{"kCompactionStyleFIFO", kCompactionStyleFIFO},
|
|
|
|
{"kCompactionStyleNone", kCompactionStyleNone}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, CompactionPri>
|
|
|
|
OptionsHelper::compaction_pri_string_map = {
|
|
|
|
{"kByCompensatedSize", kByCompensatedSize},
|
|
|
|
{"kOldestLargestSeqFirst", kOldestLargestSeqFirst},
|
|
|
|
{"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst},
|
2022-06-21 18:56:53 +00:00
|
|
|
{"kMinOverlappingRatio", kMinOverlappingRatio},
|
|
|
|
{"kRoundRobin", kRoundRobin}};
|
2017-11-18 01:02:13 +00:00
|
|
|
|
2017-12-11 21:12:12 +00:00
|
|
|
std::unordered_map<std::string, CompactionStopStyle>
|
|
|
|
OptionsHelper::compaction_stop_style_string_map = {
|
|
|
|
{"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
|
|
|
|
{"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
|
|
|
|
|
2022-01-25 22:58:48 +00:00
|
|
|
std::unordered_map<std::string, Temperature>
|
|
|
|
OptionsHelper::temperature_string_map = {
|
|
|
|
{"kUnknown", Temperature::kUnknown},
|
|
|
|
{"kHot", Temperature::kHot},
|
|
|
|
{"kWarm", Temperature::kWarm},
|
|
|
|
{"kCold", Temperature::kCold}};
|
|
|
|
|
2022-07-17 14:13:59 +00:00
|
|
|
std::unordered_map<std::string, PrepopulateBlobCache>
|
|
|
|
OptionsHelper::prepopulate_blob_cache_string_map = {
|
|
|
|
{"kDisable", PrepopulateBlobCache::kDisable},
|
|
|
|
{"kFlushOnly", PrepopulateBlobCache::kFlushOnly}};
|
|
|
|
|
2020-06-03 19:19:54 +00:00
|
|
|
Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter,
|
|
|
|
size_t pos, size_t* end, std::string* token) {
|
|
|
|
while (pos < opts.size() && isspace(opts[pos])) {
|
|
|
|
++pos;
|
|
|
|
}
|
|
|
|
// Empty value at the end
|
|
|
|
if (pos >= opts.size()) {
|
|
|
|
*token = "";
|
|
|
|
*end = std::string::npos;
|
|
|
|
return Status::OK();
|
|
|
|
} else if (opts[pos] == '{') {
|
|
|
|
int count = 1;
|
|
|
|
size_t brace_pos = pos + 1;
|
|
|
|
while (brace_pos < opts.size()) {
|
|
|
|
if (opts[brace_pos] == '{') {
|
|
|
|
++count;
|
|
|
|
} else if (opts[brace_pos] == '}') {
|
|
|
|
--count;
|
|
|
|
if (count == 0) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
++brace_pos;
|
|
|
|
}
|
|
|
|
// found the matching closing brace
|
|
|
|
if (count == 0) {
|
|
|
|
*token = trim(opts.substr(pos + 1, brace_pos - pos - 1));
|
|
|
|
// skip all whitespace and move to the next delimiter
|
|
|
|
// brace_pos points to the next position after the matching '}'
|
|
|
|
pos = brace_pos + 1;
|
|
|
|
while (pos < opts.size() && isspace(opts[pos])) {
|
|
|
|
++pos;
|
|
|
|
}
|
|
|
|
if (pos < opts.size() && opts[pos] != delimiter) {
|
|
|
|
return Status::InvalidArgument("Unexpected chars after nested options");
|
|
|
|
}
|
|
|
|
*end = pos;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Mismatched curly braces for nested options");
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
*end = opts.find(delimiter, pos);
|
|
|
|
if (*end == std::string::npos) {
|
|
|
|
// It either ends with a trailing semi-colon or the last key-value pair
|
|
|
|
*token = trim(opts.substr(pos));
|
|
|
|
} else {
|
|
|
|
*token = trim(opts.substr(pos, *end - pos));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
2020-09-14 23:59:00 +00:00
|
|
|
const std::string& value, void* opt_ptr) const {
|
2020-04-29 01:02:11 +00:00
|
|
|
if (IsDeprecated()) {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
try {
|
2020-09-14 23:59:00 +00:00
|
|
|
const std::string& opt_value = config_options.input_strings_escaped
|
|
|
|
? UnescapeOptionString(value)
|
|
|
|
: value;
|
|
|
|
|
2022-05-13 11:57:08 +00:00
|
|
|
if (opt_ptr == nullptr) {
|
2020-08-06 22:16:50 +00:00
|
|
|
return Status::NotFound("Could not find option", opt_name);
|
2020-05-21 17:56:40 +00:00
|
|
|
} else if (parse_func_ != nullptr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
ConfigOptions copy = config_options;
|
|
|
|
copy.invoke_prepare_options = false;
|
2022-05-13 11:57:08 +00:00
|
|
|
void* opt_addr = GetOffset(opt_ptr);
|
2020-09-14 23:59:00 +00:00
|
|
|
return parse_func_(copy, opt_name, opt_value, opt_addr);
|
2022-05-13 11:57:08 +00:00
|
|
|
} else if (ParseOptionHelper(GetOffset(opt_ptr), type_, opt_value)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return Status::OK();
|
2020-09-14 23:59:00 +00:00
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
// The option is <config>.<name>
|
|
|
|
Configurable* config = AsRawPointer<Configurable>(opt_ptr);
|
|
|
|
if (opt_value.empty()) {
|
|
|
|
return Status::OK();
|
|
|
|
} else if (config == nullptr) {
|
|
|
|
return Status::NotFound("Could not find configurable: ", opt_name);
|
|
|
|
} else {
|
|
|
|
ConfigOptions copy = config_options;
|
|
|
|
copy.ignore_unknown_options = false;
|
|
|
|
copy.invoke_prepare_options = false;
|
2023-12-01 19:10:30 +00:00
|
|
|
if (opt_value.find('=') != std::string::npos) {
|
2020-09-14 23:59:00 +00:00
|
|
|
return config->ConfigureFromString(copy, opt_value);
|
|
|
|
} else {
|
|
|
|
return config->ConfigureOption(copy, opt_name, opt_value);
|
|
|
|
}
|
|
|
|
}
|
2020-04-29 01:02:11 +00:00
|
|
|
} else if (IsByName()) {
|
|
|
|
return Status::NotSupported("Deserializing the option " + opt_name +
|
|
|
|
" is not supported");
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument("Error parsing:", opt_name);
|
|
|
|
}
|
|
|
|
} catch (std::exception& e) {
|
|
|
|
return Status::InvalidArgument("Error parsing " + opt_name + ":" +
|
|
|
|
std::string(e.what()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-11 23:14:33 +00:00
|
|
|
Status OptionTypeInfo::ParseType(
|
|
|
|
const ConfigOptions& config_options, const std::string& opts_str,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
void* opt_addr, std::unordered_map<std::string, std::string>* unused) {
|
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
Status status = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
} else {
|
|
|
|
return ParseType(config_options, opts_map, type_map, opt_addr, unused);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::ParseType(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
void* opt_addr, std::unordered_map<std::string, std::string>* unused) {
|
|
|
|
for (const auto& opts_iter : opts_map) {
|
|
|
|
std::string opt_name;
|
|
|
|
const auto* opt_info = Find(opts_iter.first, type_map, &opt_name);
|
|
|
|
if (opt_info != nullptr) {
|
|
|
|
Status status =
|
|
|
|
opt_info->Parse(config_options, opt_name, opts_iter.second, opt_addr);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
} else if (unused != nullptr) {
|
|
|
|
(*unused)[opts_iter.first] = opts_iter.second;
|
|
|
|
} else if (!config_options.ignore_unknown_options) {
|
|
|
|
return Status::NotFound("Unrecognized option", opts_iter.first);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
Status OptionTypeInfo::ParseStruct(
|
|
|
|
const ConfigOptions& config_options, const std::string& struct_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
const std::string& opt_name, const std::string& opt_value, void* opt_addr) {
|
2020-05-21 17:56:40 +00:00
|
|
|
assert(struct_map);
|
|
|
|
Status status;
|
|
|
|
if (opt_name == struct_name || EndsWith(opt_name, "." + struct_name)) {
|
|
|
|
// This option represents the entire struct
|
2021-05-11 23:14:33 +00:00
|
|
|
std::unordered_map<std::string, std::string> unused;
|
|
|
|
status =
|
|
|
|
ParseType(config_options, opt_value, *struct_map, opt_addr, &unused);
|
|
|
|
if (status.ok() && !unused.empty()) {
|
|
|
|
status = Status::InvalidArgument(
|
|
|
|
"Unrecognized option", struct_name + "." + unused.begin()->first);
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
|
|
|
} else if (StartsWith(opt_name, struct_name + ".")) {
|
|
|
|
// This option represents a nested field in the struct (e.g, struct.field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info =
|
|
|
|
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
|
|
|
|
if (opt_info != nullptr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
|
2020-05-21 17:56:40 +00:00
|
|
|
} else {
|
2020-08-06 22:16:50 +00:00
|
|
|
status = Status::InvalidArgument("Unrecognized option", opt_name);
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// This option represents a field in the struct (e.g. field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info = Find(opt_name, *struct_map, &elem_name);
|
|
|
|
if (opt_info != nullptr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
|
2020-05-21 17:56:40 +00:00
|
|
|
} else {
|
2020-08-06 22:16:50 +00:00
|
|
|
status = Status::InvalidArgument("Unrecognized option",
|
2020-05-21 17:56:40 +00:00
|
|
|
struct_name + "." + opt_name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
2020-09-14 23:59:00 +00:00
|
|
|
const void* const opt_ptr,
|
2020-05-21 17:56:40 +00:00
|
|
|
std::string* opt_value) const {
|
2020-04-29 01:02:11 +00:00
|
|
|
// If the option is no longer used in rocksdb and marked as deprecated,
|
|
|
|
// we skip it in the serialization.
|
2022-05-13 11:57:08 +00:00
|
|
|
if (opt_ptr == nullptr || IsDeprecated()) {
|
2020-09-14 23:59:00 +00:00
|
|
|
return Status::OK();
|
|
|
|
} else if (IsEnabled(OptionTypeFlags::kDontSerialize)) {
|
|
|
|
return Status::NotSupported("Cannot serialize option: ", opt_name);
|
|
|
|
} else if (serialize_func_ != nullptr) {
|
2022-05-13 11:57:08 +00:00
|
|
|
const void* opt_addr = GetOffset(opt_ptr);
|
2020-09-14 23:59:00 +00:00
|
|
|
return serialize_func_(config_options, opt_name, opt_addr, opt_value);
|
2020-11-11 23:09:14 +00:00
|
|
|
} else if (IsCustomizable()) {
|
|
|
|
const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
|
2021-09-27 14:42:36 +00:00
|
|
|
opt_value->clear();
|
2020-11-11 23:09:14 +00:00
|
|
|
if (custom == nullptr) {
|
2021-08-19 17:09:30 +00:00
|
|
|
// We do not have a custom object to serialize.
|
|
|
|
// If the option is not mutable and we are doing only mutable options,
|
|
|
|
// we return an empty string (which will cause the option not to be
|
|
|
|
// printed). Otherwise, we return the "nullptr" string, which will result
|
|
|
|
// in "option=nullptr" being printed.
|
|
|
|
if (IsMutable() || !config_options.mutable_options_only) {
|
|
|
|
*opt_value = kNullptrString;
|
|
|
|
} else {
|
|
|
|
*opt_value = "";
|
|
|
|
}
|
2020-11-11 23:09:14 +00:00
|
|
|
} else if (IsEnabled(OptionTypeFlags::kStringNameOnly) &&
|
|
|
|
!config_options.IsDetailed()) {
|
2021-09-27 14:42:36 +00:00
|
|
|
if (!config_options.mutable_options_only || IsMutable()) {
|
|
|
|
*opt_value = custom->GetId();
|
|
|
|
}
|
2020-11-11 23:09:14 +00:00
|
|
|
} else {
|
|
|
|
ConfigOptions embedded = config_options;
|
|
|
|
embedded.delimiter = ";";
|
2021-06-28 19:27:39 +00:00
|
|
|
// If this option is mutable, everything inside it should be considered
|
|
|
|
// mutable
|
|
|
|
if (IsMutable()) {
|
|
|
|
embedded.mutable_options_only = false;
|
|
|
|
}
|
|
|
|
std::string value = custom->ToString(embedded);
|
|
|
|
if (!embedded.mutable_options_only ||
|
2023-12-01 19:10:30 +00:00
|
|
|
value.find('=') != std::string::npos) {
|
2021-06-28 19:27:39 +00:00
|
|
|
*opt_value = value;
|
|
|
|
} else {
|
|
|
|
*opt_value = "";
|
|
|
|
}
|
2020-11-11 23:09:14 +00:00
|
|
|
}
|
|
|
|
return Status::OK();
|
2020-09-14 23:59:00 +00:00
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
|
|
|
|
if (config != nullptr) {
|
|
|
|
ConfigOptions embedded = config_options;
|
|
|
|
embedded.delimiter = ";";
|
|
|
|
*opt_value = config->ToString(embedded);
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
2020-09-14 23:59:00 +00:00
|
|
|
return Status::OK();
|
2021-06-28 19:27:39 +00:00
|
|
|
} else if (config_options.mutable_options_only && !IsMutable()) {
|
|
|
|
return Status::OK();
|
2022-05-13 11:57:08 +00:00
|
|
|
} else if (SerializeSingleOptionHelper(GetOffset(opt_ptr), type_,
|
|
|
|
opt_value)) {
|
2021-06-28 19:27:39 +00:00
|
|
|
return Status::OK();
|
2020-09-14 23:59:00 +00:00
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument("Cannot serialize option: ", opt_name);
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-11 23:14:33 +00:00
|
|
|
Status OptionTypeInfo::SerializeType(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
const void* opt_addr, std::string* result) {
|
|
|
|
Status status;
|
|
|
|
for (const auto& iter : type_map) {
|
|
|
|
std::string single;
|
|
|
|
const auto& opt_info = iter.second;
|
|
|
|
if (opt_info.ShouldSerialize()) {
|
|
|
|
status =
|
|
|
|
opt_info.Serialize(config_options, iter.first, opt_addr, &single);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
} else {
|
|
|
|
result->append(iter.first + "=" + single + config_options.delimiter);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
Status OptionTypeInfo::SerializeStruct(
|
|
|
|
const ConfigOptions& config_options, const std::string& struct_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
const std::string& opt_name, const void* opt_addr, std::string* value) {
|
2020-05-21 17:56:40 +00:00
|
|
|
assert(struct_map);
|
|
|
|
Status status;
|
|
|
|
if (EndsWith(opt_name, struct_name)) {
|
|
|
|
// We are going to write the struct as "{ prop1=value1; prop2=value2;}.
|
|
|
|
// Set the delimiter to ";" so that the everything will be on one line.
|
|
|
|
ConfigOptions embedded = config_options;
|
|
|
|
embedded.delimiter = ";";
|
|
|
|
|
|
|
|
// This option represents the entire struct
|
|
|
|
std::string result;
|
2021-05-11 23:14:33 +00:00
|
|
|
status = SerializeType(embedded, *struct_map, opt_addr, &result);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
} else {
|
|
|
|
*value = "{" + result + "}";
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
|
|
|
} else if (StartsWith(opt_name, struct_name + ".")) {
|
|
|
|
// This option represents a nested field in the struct (e.g, struct.field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info =
|
|
|
|
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
|
|
|
|
if (opt_info != nullptr) {
|
2020-09-14 23:59:00 +00:00
|
|
|
status = opt_info->Serialize(config_options, elem_name, opt_addr, value);
|
2020-05-21 17:56:40 +00:00
|
|
|
} else {
|
2020-08-06 22:16:50 +00:00
|
|
|
status = Status::InvalidArgument("Unrecognized option", opt_name);
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
2020-04-29 01:02:11 +00:00
|
|
|
} else {
|
2020-05-21 17:56:40 +00:00
|
|
|
// This option represents a field in the struct (e.g. field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info = Find(opt_name, *struct_map, &elem_name);
|
|
|
|
if (opt_info == nullptr) {
|
2020-08-06 22:16:50 +00:00
|
|
|
status = Status::InvalidArgument("Unrecognized option", opt_name);
|
2020-05-21 17:56:40 +00:00
|
|
|
} else if (opt_info->ShouldSerialize()) {
|
2020-05-22 18:15:44 +00:00
|
|
|
status = opt_info->Serialize(config_options, opt_name + "." + elem_name,
|
2020-09-14 23:59:00 +00:00
|
|
|
opt_addr, value);
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
2020-05-22 18:15:44 +00:00
|
|
|
return status;
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template <typename T>
|
2021-05-13 21:28:50 +00:00
|
|
|
bool IsOptionEqual(const void* offset1, const void* offset2) {
|
|
|
|
return (*static_cast<const T*>(offset1) == *static_cast<const T*>(offset2));
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool AreEqualDoubles(const double a, const double b) {
|
|
|
|
return (fabs(a - b) < 0.00001);
|
|
|
|
}
|
|
|
|
|
2021-05-13 21:28:50 +00:00
|
|
|
static bool AreOptionsEqual(OptionType type, const void* this_offset,
|
|
|
|
const void* that_offset) {
|
2020-04-29 01:02:11 +00:00
|
|
|
switch (type) {
|
|
|
|
case OptionType::kBoolean:
|
|
|
|
return IsOptionEqual<bool>(this_offset, that_offset);
|
|
|
|
case OptionType::kInt:
|
|
|
|
return IsOptionEqual<int>(this_offset, that_offset);
|
|
|
|
case OptionType::kUInt:
|
|
|
|
return IsOptionEqual<unsigned int>(this_offset, that_offset);
|
|
|
|
case OptionType::kInt32T:
|
|
|
|
return IsOptionEqual<int32_t>(this_offset, that_offset);
|
|
|
|
case OptionType::kInt64T: {
|
|
|
|
int64_t v1, v2;
|
2021-05-13 21:28:50 +00:00
|
|
|
GetUnaligned(static_cast<const int64_t*>(this_offset), &v1);
|
|
|
|
GetUnaligned(static_cast<const int64_t*>(that_offset), &v2);
|
2020-04-29 01:02:11 +00:00
|
|
|
return (v1 == v2);
|
|
|
|
}
|
2021-05-20 04:40:43 +00:00
|
|
|
case OptionType::kUInt8T:
|
|
|
|
return IsOptionEqual<uint8_t>(this_offset, that_offset);
|
2020-04-29 01:02:11 +00:00
|
|
|
case OptionType::kUInt32T:
|
|
|
|
return IsOptionEqual<uint32_t>(this_offset, that_offset);
|
|
|
|
case OptionType::kUInt64T: {
|
|
|
|
uint64_t v1, v2;
|
2021-05-13 21:28:50 +00:00
|
|
|
GetUnaligned(static_cast<const uint64_t*>(this_offset), &v1);
|
|
|
|
GetUnaligned(static_cast<const uint64_t*>(that_offset), &v2);
|
2020-04-29 01:02:11 +00:00
|
|
|
return (v1 == v2);
|
|
|
|
}
|
|
|
|
case OptionType::kSizeT: {
|
|
|
|
size_t v1, v2;
|
2021-05-13 21:28:50 +00:00
|
|
|
GetUnaligned(static_cast<const size_t*>(this_offset), &v1);
|
|
|
|
GetUnaligned(static_cast<const size_t*>(that_offset), &v2);
|
2020-04-29 01:02:11 +00:00
|
|
|
return (v1 == v2);
|
|
|
|
}
|
2023-09-15 22:46:10 +00:00
|
|
|
case OptionType::kAtomicInt:
|
|
|
|
return IsOptionEqual<std::atomic<int>>(this_offset, that_offset);
|
2020-04-29 01:02:11 +00:00
|
|
|
case OptionType::kString:
|
|
|
|
return IsOptionEqual<std::string>(this_offset, that_offset);
|
|
|
|
case OptionType::kDouble:
|
2021-05-13 21:28:50 +00:00
|
|
|
return AreEqualDoubles(*static_cast<const double*>(this_offset),
|
|
|
|
*static_cast<const double*>(that_offset));
|
2020-04-29 01:02:11 +00:00
|
|
|
case OptionType::kCompactionStyle:
|
|
|
|
return IsOptionEqual<CompactionStyle>(this_offset, that_offset);
|
|
|
|
case OptionType::kCompactionStopStyle:
|
|
|
|
return IsOptionEqual<CompactionStopStyle>(this_offset, that_offset);
|
|
|
|
case OptionType::kCompactionPri:
|
|
|
|
return IsOptionEqual<CompactionPri>(this_offset, that_offset);
|
|
|
|
case OptionType::kCompressionType:
|
|
|
|
return IsOptionEqual<CompressionType>(this_offset, that_offset);
|
|
|
|
case OptionType::kChecksumType:
|
|
|
|
return IsOptionEqual<ChecksumType>(this_offset, that_offset);
|
|
|
|
case OptionType::kEncodingType:
|
|
|
|
return IsOptionEqual<EncodingType>(this_offset, that_offset);
|
2021-05-12 19:34:22 +00:00
|
|
|
case OptionType::kEncodedString:
|
|
|
|
return IsOptionEqual<std::string>(this_offset, that_offset);
|
2022-01-25 22:58:48 +00:00
|
|
|
case OptionType::kTemperature:
|
|
|
|
return IsOptionEqual<Temperature>(this_offset, that_offset);
|
2020-04-29 01:02:11 +00:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
} // End switch
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
2020-09-14 23:59:00 +00:00
|
|
|
const void* const this_ptr,
|
|
|
|
const void* const that_ptr,
|
2020-05-21 17:56:40 +00:00
|
|
|
std::string* mismatch) const {
|
2020-09-14 23:59:00 +00:00
|
|
|
auto level = GetSanityLevel();
|
|
|
|
if (!config_options.IsCheckEnabled(level)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return true; // If the sanity level is not being checked, skip it
|
|
|
|
}
|
2022-05-13 11:57:08 +00:00
|
|
|
if (this_ptr == nullptr || that_ptr == nullptr) {
|
|
|
|
if (this_ptr == that_ptr) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return true;
|
|
|
|
}
|
2020-05-21 17:56:40 +00:00
|
|
|
} else if (equals_func_ != nullptr) {
|
2022-05-13 11:57:08 +00:00
|
|
|
const void* this_addr = GetOffset(this_ptr);
|
|
|
|
const void* that_addr = GetOffset(that_ptr);
|
2020-05-21 17:56:40 +00:00
|
|
|
if (equals_func_(config_options, opt_name, this_addr, that_addr,
|
|
|
|
mismatch)) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return true;
|
|
|
|
}
|
2022-05-13 11:57:08 +00:00
|
|
|
} else {
|
|
|
|
const void* this_addr = GetOffset(this_ptr);
|
|
|
|
const void* that_addr = GetOffset(that_ptr);
|
|
|
|
if (AreOptionsEqual(type_, this_addr, that_addr)) {
|
2020-09-14 23:59:00 +00:00
|
|
|
return true;
|
2022-05-13 11:57:08 +00:00
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
const auto* this_config = AsRawPointer<Configurable>(this_ptr);
|
|
|
|
const auto* that_config = AsRawPointer<Configurable>(that_ptr);
|
|
|
|
if (this_config == that_config) {
|
|
|
|
return true;
|
|
|
|
} else if (this_config != nullptr && that_config != nullptr) {
|
|
|
|
std::string bad_name;
|
|
|
|
bool matches;
|
|
|
|
if (level < config_options.sanity_level) {
|
|
|
|
ConfigOptions copy = config_options;
|
|
|
|
copy.sanity_level = level;
|
|
|
|
matches = this_config->AreEquivalent(copy, that_config, &bad_name);
|
|
|
|
} else {
|
|
|
|
matches = this_config->AreEquivalent(config_options, that_config,
|
|
|
|
&bad_name);
|
|
|
|
}
|
|
|
|
if (!matches) {
|
|
|
|
*mismatch = opt_name + "." + bad_name;
|
|
|
|
}
|
|
|
|
return matches;
|
2020-09-14 23:59:00 +00:00
|
|
|
}
|
|
|
|
}
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
|
|
|
if (mismatch->empty()) {
|
|
|
|
*mismatch = opt_name;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2021-05-11 23:14:33 +00:00
|
|
|
bool OptionTypeInfo::TypesAreEqual(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
const void* this_addr, const void* that_addr, std::string* mismatch) {
|
|
|
|
for (const auto& iter : type_map) {
|
|
|
|
const auto& opt_info = iter.second;
|
|
|
|
if (!opt_info.AreEqual(config_options, iter.first, this_addr, that_addr,
|
|
|
|
mismatch)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
bool OptionTypeInfo::StructsAreEqual(
|
|
|
|
const ConfigOptions& config_options, const std::string& struct_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
|
2021-05-13 21:28:50 +00:00
|
|
|
const std::string& opt_name, const void* this_addr, const void* that_addr,
|
2020-05-21 17:56:40 +00:00
|
|
|
std::string* mismatch) {
|
|
|
|
assert(struct_map);
|
|
|
|
bool matches = true;
|
|
|
|
std::string result;
|
|
|
|
if (EndsWith(opt_name, struct_name)) {
|
|
|
|
// This option represents the entire struct
|
2021-05-11 23:14:33 +00:00
|
|
|
matches = TypesAreEqual(config_options, *struct_map, this_addr, that_addr,
|
|
|
|
&result);
|
|
|
|
if (!matches) {
|
|
|
|
*mismatch = struct_name + "." + result;
|
|
|
|
return false;
|
2020-05-21 17:56:40 +00:00
|
|
|
}
|
|
|
|
} else if (StartsWith(opt_name, struct_name + ".")) {
|
|
|
|
// This option represents a nested field in the struct (e.g, struct.field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info =
|
|
|
|
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
|
|
|
|
assert(opt_info);
|
|
|
|
if (opt_info == nullptr) {
|
|
|
|
*mismatch = opt_name;
|
|
|
|
matches = false;
|
2020-09-14 23:59:00 +00:00
|
|
|
} else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
|
|
|
|
that_addr, &result)) {
|
2020-05-21 17:56:40 +00:00
|
|
|
matches = false;
|
|
|
|
*mismatch = struct_name + "." + result;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// This option represents a field in the struct (e.g. field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info = Find(opt_name, *struct_map, &elem_name);
|
|
|
|
assert(opt_info);
|
|
|
|
if (opt_info == nullptr) {
|
|
|
|
*mismatch = struct_name + "." + opt_name;
|
|
|
|
matches = false;
|
2020-09-14 23:59:00 +00:00
|
|
|
} else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
|
|
|
|
that_addr, &result)) {
|
2020-05-21 17:56:40 +00:00
|
|
|
matches = false;
|
|
|
|
*mismatch = struct_name + "." + result;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return matches;
|
|
|
|
}
|
|
|
|
|
2020-09-14 23:59:00 +00:00
|
|
|
bool MatchesOptionsTypeFromMap(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
const void* const this_ptr, const void* const that_ptr,
|
|
|
|
std::string* mismatch) {
|
|
|
|
for (auto& pair : type_map) {
|
|
|
|
// We skip checking deprecated variables as they might
|
|
|
|
// contain random values since they might not be initialized
|
|
|
|
if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) {
|
|
|
|
if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr,
|
|
|
|
mismatch) &&
|
|
|
|
!pair.second.AreEqualByName(config_options, pair.first, this_ptr,
|
|
|
|
that_ptr)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
2020-09-14 23:59:00 +00:00
|
|
|
const void* const this_ptr,
|
|
|
|
const void* const that_ptr) const {
|
2020-04-29 01:02:11 +00:00
|
|
|
if (IsByName()) {
|
|
|
|
std::string that_value;
|
2020-09-14 23:59:00 +00:00
|
|
|
if (Serialize(config_options, opt_name, that_ptr, &that_value).ok()) {
|
|
|
|
return AreEqualByName(config_options, opt_name, this_ptr, that_value);
|
2020-04-29 01:02:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
2020-09-14 23:59:00 +00:00
|
|
|
const void* const opt_ptr,
|
2020-05-21 17:56:40 +00:00
|
|
|
const std::string& that_value) const {
|
2020-04-29 01:02:11 +00:00
|
|
|
std::string this_value;
|
|
|
|
if (!IsByName()) {
|
|
|
|
return false;
|
2020-09-14 23:59:00 +00:00
|
|
|
} else if (!Serialize(config_options, opt_name, opt_ptr, &this_value).ok()) {
|
2020-04-29 01:02:11 +00:00
|
|
|
return false;
|
|
|
|
} else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) {
|
|
|
|
if (that_value == kNullptrString) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} else if (IsEnabled(OptionVerificationType::kByNameAllowNull)) {
|
|
|
|
if (that_value == kNullptrString) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return (this_value == that_value);
|
|
|
|
}
|
2020-05-21 17:56:40 +00:00
|
|
|
|
2022-05-13 11:57:08 +00:00
|
|
|
Status OptionTypeInfo::Prepare(const ConfigOptions& config_options,
|
|
|
|
const std::string& name, void* opt_ptr) const {
|
|
|
|
if (ShouldPrepare()) {
|
|
|
|
if (prepare_func_ != nullptr) {
|
|
|
|
void* opt_addr = GetOffset(opt_ptr);
|
|
|
|
return prepare_func_(config_options, name, opt_addr);
|
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
Configurable* config = AsRawPointer<Configurable>(opt_ptr);
|
|
|
|
if (config != nullptr) {
|
|
|
|
return config->PrepareOptions(config_options);
|
|
|
|
} else if (!CanBeNull()) {
|
|
|
|
return Status::NotFound("Missing configurable object", name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::Validate(const DBOptions& db_opts,
|
|
|
|
const ColumnFamilyOptions& cf_opts,
|
|
|
|
const std::string& name,
|
|
|
|
const void* opt_ptr) const {
|
|
|
|
if (ShouldValidate()) {
|
|
|
|
if (validate_func_ != nullptr) {
|
|
|
|
const void* opt_addr = GetOffset(opt_ptr);
|
|
|
|
return validate_func_(db_opts, cf_opts, name, opt_addr);
|
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
|
|
|
|
if (config != nullptr) {
|
|
|
|
return config->ValidateOptions(db_opts, cf_opts);
|
|
|
|
} else if (!CanBeNull()) {
|
|
|
|
return Status::NotFound("Missing configurable object", name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2020-05-21 17:56:40 +00:00
|
|
|
const OptionTypeInfo* OptionTypeInfo::Find(
|
|
|
|
const std::string& opt_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& opt_map,
|
|
|
|
std::string* elem_name) {
|
|
|
|
const auto iter = opt_map.find(opt_name); // Look up the value in the map
|
|
|
|
if (iter != opt_map.end()) { // Found the option in the map
|
|
|
|
*elem_name = opt_name; // Return the name
|
|
|
|
return &(iter->second); // Return the contents of the iterator
|
|
|
|
} else {
|
2023-12-01 19:10:30 +00:00
|
|
|
auto idx = opt_name.find('.'); // Look for a separator
|
2020-05-21 17:56:40 +00:00
|
|
|
if (idx > 0 && idx != std::string::npos) { // We found a separator
|
|
|
|
auto siter =
|
|
|
|
opt_map.find(opt_name.substr(0, idx)); // Look for the short name
|
|
|
|
if (siter != opt_map.end()) { // We found the short name
|
2020-09-14 23:59:00 +00:00
|
|
|
if (siter->second.IsStruct() || // If the object is a struct
|
|
|
|
siter->second.IsConfigurable()) { // or a Configurable
|
2020-05-21 17:56:40 +00:00
|
|
|
*elem_name = opt_name.substr(idx + 1); // Return the rest
|
|
|
|
return &(siter->second); // Return the contents of the iterator
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nullptr;
|
|
|
|
}
|
2016-09-15 05:10:28 +00:00
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|